575 lines
19 KiB
TypeScript
575 lines
19 KiB
TypeScript
import { randomUUID } from 'node:crypto'
|
|
import { readFile } from 'node:fs/promises'
|
|
import path from 'node:path'
|
|
|
|
import {
|
|
authorizeWorkspaceAction,
|
|
completeFirstRun,
|
|
composeAndCreateGeneratedRun,
|
|
createGeneratedArtifact,
|
|
downloadGeneratedArtifact,
|
|
getGeneratedRun,
|
|
type ImmutableJsonObject,
|
|
} from '../../packages/application/src/index'
|
|
import { LocalArtifactStorage } from '../../packages/artifacts/src/index'
|
|
import type {
|
|
CanonicalPromptRequest,
|
|
PlaybookMetadata,
|
|
PlaybookSpecification,
|
|
RepositoryProfile,
|
|
} from '../../packages/composer/src/index'
|
|
import {
|
|
canonicalJson,
|
|
loadBuiltInPlaybookRecords,
|
|
sha256,
|
|
} from '../../packages/content/src/index'
|
|
import {
|
|
closeDatabase,
|
|
DrizzleFirstRunStore,
|
|
DrizzleFirstRunTransactionRunner,
|
|
DrizzleGeneratedArtifactStore,
|
|
DrizzleGeneratedRunStore,
|
|
DrizzlePlaybookCatalog,
|
|
DrizzleWorkspaceAuthorizationLookup,
|
|
getPersistedInstanceStatus,
|
|
getSqlClient,
|
|
} from '../../packages/db/src/index'
|
|
import { afterAll, describe, expect, it } from 'vitest'
|
|
import { parse } from 'yaml'
|
|
|
|
const exampleProfile: RepositoryProfile = {
|
|
metadata: { name: 'Example TypeScript Service', revision: 1 },
|
|
spec: {
|
|
repositoryType: 'single-app',
|
|
stack: {
|
|
languages: ['TypeScript'],
|
|
frameworks: ['Next.js'],
|
|
packageManagers: ['pnpm'],
|
|
databases: ['PostgreSQL'],
|
|
deploymentTypes: ['Docker Compose'],
|
|
},
|
|
commands: [
|
|
['lint', 'pnpm lint'],
|
|
['typecheck', 'pnpm typecheck'],
|
|
['unit-test', 'pnpm test'],
|
|
['build', 'pnpm build'],
|
|
].map(([role, command]) => ({
|
|
role: role!,
|
|
command: command!,
|
|
workingDirectory: '.',
|
|
})),
|
|
paths: {
|
|
applicationRoots: ['apps/web', 'packages'],
|
|
testRoots: ['tests', 'apps/web/tests'],
|
|
documentationRoots: ['docs'],
|
|
protected: ['data', 'backups', '.env'],
|
|
excluded: ['node_modules', '.git'],
|
|
},
|
|
policies: {
|
|
preserveBackwardCompatibility: true,
|
|
newDependencies: 'justify',
|
|
gitWrite: 'none',
|
|
migrations: 'reversible-only',
|
|
productionDataAccess: 'forbidden',
|
|
},
|
|
},
|
|
}
|
|
|
|
async function loadGovernedExampleProfile(): Promise<ImmutableJsonObject> {
|
|
return parse(
|
|
await readFile(
|
|
path.resolve('examples/repository-profiles/example-profile.yaml'),
|
|
'utf8',
|
|
),
|
|
) as ImmutableJsonObject
|
|
}
|
|
|
|
let ownerId = ''
|
|
let primaryWorkspaceId = ''
|
|
let generatedRunId = ''
|
|
let generatedArtifactId = ''
|
|
|
|
afterAll(async () => closeDatabase())
|
|
|
|
const databaseIntegration = process.env.DATABASE_URL ? describe : describe.skip
|
|
|
|
databaseIntegration('Milestone 0 service-backed vertical slice', () => {
|
|
it('atomically initializes, imports, composes, persists and protects the golden task', async () => {
|
|
await expect(getPersistedInstanceStatus()).resolves.toMatchObject({
|
|
state: 'uninitialized',
|
|
setupRequired: true,
|
|
})
|
|
|
|
const records = await loadBuiltInPlaybookRecords()
|
|
const setup = await completeFirstRun(new DrizzleFirstRunStore(records), {
|
|
instanceName: 'Integration instance',
|
|
publicBaseUrl: 'http://127.0.0.1:3000',
|
|
owner: {
|
|
email: 'owner@integration.test',
|
|
displayName: 'Integration owner',
|
|
passwordHash: 'better-auth-prehashed-integration-fixture',
|
|
},
|
|
configuration: { instanceName: 'Integration instance' },
|
|
configurationDigest: sha256(
|
|
canonicalJson({ instanceName: 'Integration instance' }),
|
|
),
|
|
})
|
|
ownerId = setup.ownerId
|
|
primaryWorkspaceId = setup.workspaceId
|
|
|
|
const sql = getSqlClient()
|
|
const [catalogCount] = await sql<{ count: number }[]>`
|
|
select count(*)::int as count from playbook_versions
|
|
`
|
|
expect(catalogCount?.count).toBe(28)
|
|
await expect(getPersistedInstanceStatus()).resolves.toMatchObject({
|
|
state: 'ready',
|
|
setupRequired: false,
|
|
})
|
|
|
|
const rootCause = records.find(
|
|
(record) => record.slug === 'root-cause-bugfix',
|
|
)
|
|
expect(rootCause).toBeDefined()
|
|
const [version] = await sql<{ id: string }[]>`
|
|
select pv.id
|
|
from playbook_versions pv
|
|
join playbooks p on p.id = pv.playbook_id
|
|
where p.slug = 'root-cause-bugfix' and pv.semantic_version = '1.0.0'
|
|
`
|
|
expect(version?.id).toBeTruthy()
|
|
|
|
const manifest = rootCause!.packageJson as unknown as {
|
|
metadata: PlaybookMetadata
|
|
spec: PlaybookSpecification
|
|
}
|
|
const inputs = {
|
|
problemStatement: 'Example value for Problem statement',
|
|
reproductionClues: '',
|
|
preserveCompatibility: true,
|
|
affectedScope: [],
|
|
}
|
|
const prompt: CanonicalPromptRequest = {
|
|
metadata: manifest.metadata,
|
|
specification: manifest.spec,
|
|
template: rootCause!.templateText,
|
|
inputs,
|
|
workMode: 'guided',
|
|
autonomyLevel: 'verify',
|
|
repositoryProfile: exampleProfile,
|
|
}
|
|
const request = {
|
|
prompt,
|
|
snapshots: {
|
|
playbook: rootCause!.packageJson as unknown as ImmutableJsonObject,
|
|
repositoryProfile: await loadGovernedExampleProfile(),
|
|
normalizedInput: inputs,
|
|
policy: { autonomyLevel: 'verify', conditionsResolved: true },
|
|
provenance: [],
|
|
},
|
|
lint: { exportReadiness: 'ready' as const, findings: [] },
|
|
generatedBy: setup.ownerId,
|
|
workspaceId: setup.workspaceId,
|
|
playbookVersionId: version!.id,
|
|
idempotencyKey: 'milestone-zero-root-cause-golden',
|
|
}
|
|
const dependencies = {
|
|
store: new DrizzleGeneratedRunStore(),
|
|
nextId: randomUUID,
|
|
now: () => new Date(),
|
|
workspaceAuthorization: new DrizzleWorkspaceAuthorizationLookup(),
|
|
}
|
|
const created = await composeAndCreateGeneratedRun(dependencies, request)
|
|
const golden = await readFile(
|
|
path.resolve('examples/rendered-prompts/root-cause-bugfix.md'),
|
|
'utf8',
|
|
)
|
|
expect(created.created).toBe(true)
|
|
expect(created.run.renderedPrompt).toBe(golden)
|
|
expect(created.run.renderDigest).toBe(sha256(golden))
|
|
generatedRunId = created.run.id
|
|
generatedArtifactId = randomUUID()
|
|
const artifactRoot = process.env.ARTIFACT_ROOT
|
|
if (!artifactRoot)
|
|
throw new Error('ARTIFACT_ROOT is required for integration tests')
|
|
const artifactBytes = new TextEncoder().encode(golden)
|
|
const artifact = await createGeneratedArtifact(
|
|
{
|
|
authorization: new DrizzleWorkspaceAuthorizationLookup(),
|
|
metadata: new DrizzleGeneratedArtifactStore(),
|
|
storage: new LocalArtifactStorage(artifactRoot),
|
|
now: () => new Date(),
|
|
},
|
|
{
|
|
actor: { userId: ownerId },
|
|
workspaceId: primaryWorkspaceId,
|
|
artifactId: generatedArtifactId,
|
|
runId: created.run.id,
|
|
artifactType: 'prompt_text',
|
|
filename: 'root-cause-bugfix.md',
|
|
mediaType: 'text/markdown; charset=utf-8',
|
|
content: artifactBytes,
|
|
},
|
|
)
|
|
expect(artifact.created).toBe(true)
|
|
expect(artifact.artifact.sha256).toBe(sha256(golden))
|
|
|
|
const retried = await composeAndCreateGeneratedRun(dependencies, request)
|
|
expect(retried.created).toBe(false)
|
|
expect(retried.run.id).toBe(created.run.id)
|
|
|
|
await expect(
|
|
sql`update generated_runs set rendered_prompt = 'mutated' where id = ${created.run.id}`,
|
|
).rejects.toThrow()
|
|
await expect(
|
|
sql`update playbook_versions set title = 'mutated' where id = ${version!.id}`,
|
|
).rejects.toThrow(/playbook_versions is immutable/u)
|
|
|
|
const duplicateRunner = new DrizzleFirstRunTransactionRunner()
|
|
await expect(
|
|
duplicateRunner.run(records, (transaction) =>
|
|
transaction.importBuiltInPlaybooks(),
|
|
),
|
|
).resolves.toEqual({ imported: 28 })
|
|
const catalogBeforeReconnect =
|
|
await new DrizzlePlaybookCatalog().listBuiltIns()
|
|
expect(catalogBeforeReconnect).toHaveLength(28)
|
|
expect(
|
|
catalogBeforeReconnect.map(({ slug, version, digest }) => ({
|
|
slug,
|
|
version,
|
|
digest,
|
|
})),
|
|
).toEqual(
|
|
records.map((record) => ({
|
|
slug: record.slug,
|
|
version: record.semanticVersion,
|
|
digest: record.contentDigest,
|
|
})),
|
|
)
|
|
|
|
const searchableCatalog = new DrizzlePlaybookCatalog()
|
|
const searchMatches = await searchableCatalog.list({
|
|
q: 'root cause defect',
|
|
category: ['bugfixing'],
|
|
riskTier: ['moderate'],
|
|
lifecycle: ['reviewed'],
|
|
source: ['built_in'],
|
|
})
|
|
expect(searchMatches.map(({ slug }) => slug)).toEqual(['root-cause-bugfix'])
|
|
const rootCauseDetail = await searchableCatalog.findBySlug(
|
|
'root-cause-bugfix',
|
|
'built_in',
|
|
)
|
|
expect(rootCauseDetail?.current).toMatchObject({
|
|
version: '1.0.0',
|
|
digest: rootCause!.contentDigest,
|
|
manifest: { metadata: { slug: 'root-cause-bugfix' } },
|
|
})
|
|
expect(rootCauseDetail?.versions).toHaveLength(1)
|
|
await expect(
|
|
searchableCatalog.findVersionBySlug(
|
|
'root-cause-bugfix',
|
|
'1.0.0',
|
|
'built_in',
|
|
),
|
|
).resolves.toMatchObject({
|
|
template: rootCause!.templateText,
|
|
quality: rootCause!.packageJson.quality,
|
|
})
|
|
|
|
const conflictingRecords = records.map((record) =>
|
|
record.slug === 'root-cause-bugfix'
|
|
? { ...record, contentDigest: '0'.repeat(64) }
|
|
: record,
|
|
)
|
|
await expect(
|
|
duplicateRunner.run(conflictingRecords, (transaction) =>
|
|
transaction.importBuiltInPlaybooks(),
|
|
),
|
|
).rejects.toMatchObject({ code: 'playbook_version_conflict' })
|
|
|
|
await closeDatabase()
|
|
await expect(new DrizzlePlaybookCatalog().listBuiltIns()).resolves.toEqual(
|
|
catalogBeforeReconnect,
|
|
)
|
|
const restoredArtifact = await downloadGeneratedArtifact(
|
|
{
|
|
authorization: new DrizzleWorkspaceAuthorizationLookup(),
|
|
metadata: new DrizzleGeneratedArtifactStore(),
|
|
storage: new LocalArtifactStorage(artifactRoot),
|
|
},
|
|
{
|
|
actor: { userId: ownerId },
|
|
workspaceId: primaryWorkspaceId,
|
|
artifactId: generatedArtifactId,
|
|
},
|
|
)
|
|
expect(Buffer.from(restoredArtifact.content).toString('utf8')).toBe(golden)
|
|
expect(restoredArtifact.artifact.sha256).toBe(sha256(golden))
|
|
}, 30_000)
|
|
|
|
it('persists role boundaries without cross-workspace or instance-admin bypass', async () => {
|
|
expect(ownerId).toBeTruthy()
|
|
expect(primaryWorkspaceId).toBeTruthy()
|
|
const sql = getSqlClient()
|
|
const viewerId = randomUUID()
|
|
const editorId = randomUUID()
|
|
const administratorId = randomUUID()
|
|
const isolatedWorkspaceId = randomUUID()
|
|
|
|
await sql`
|
|
insert into users (
|
|
id, email, display_name, password_hash, email_verified,
|
|
instance_role, status
|
|
) values
|
|
(${viewerId}, 'viewer@integration.test', 'Integration viewer', 'not-a-login-credential', true, 'user', 'active'),
|
|
(${editorId}, 'editor@integration.test', 'Integration editor', 'not-a-login-credential', true, 'user', 'active'),
|
|
(${administratorId}, 'admin@integration.test', 'Integration administrator', 'not-a-login-credential', true, 'instance_admin', 'active')
|
|
`
|
|
await sql`
|
|
insert into workspaces (id, name, type)
|
|
values (${isolatedWorkspaceId}, 'Isolated integration workspace', 'team')
|
|
`
|
|
await sql`
|
|
insert into workspace_memberships (workspace_id, user_id, role)
|
|
values
|
|
(${primaryWorkspaceId}, ${viewerId}, 'viewer'),
|
|
(${primaryWorkspaceId}, ${editorId}, 'editor')
|
|
`
|
|
|
|
const lookup = new DrizzleWorkspaceAuthorizationLookup()
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: viewerId },
|
|
workspaceId: primaryWorkspaceId,
|
|
action: 'read',
|
|
}),
|
|
).resolves.toMatchObject({ workspaceRole: 'viewer' })
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: viewerId },
|
|
workspaceId: primaryWorkspaceId,
|
|
action: 'write',
|
|
}),
|
|
).rejects.toMatchObject({ code: 'workspace_access_denied' })
|
|
const reader = new DrizzleGeneratedRunStore()
|
|
await expect(
|
|
getGeneratedRun(
|
|
{ reader, workspaceAuthorization: lookup },
|
|
{
|
|
actor: { userId: viewerId },
|
|
workspaceId: primaryWorkspaceId,
|
|
runId: generatedRunId,
|
|
},
|
|
),
|
|
).resolves.toMatchObject({ id: generatedRunId })
|
|
await expect(
|
|
getGeneratedRun(
|
|
{ reader, workspaceAuthorization: lookup },
|
|
{
|
|
actor: { userId: viewerId },
|
|
workspaceId: isolatedWorkspaceId,
|
|
runId: generatedRunId,
|
|
},
|
|
),
|
|
).rejects.toMatchObject({ code: 'workspace_access_denied' })
|
|
const artifactRoot = process.env.ARTIFACT_ROOT
|
|
if (!artifactRoot)
|
|
throw new Error('ARTIFACT_ROOT is required for integration tests')
|
|
await expect(
|
|
downloadGeneratedArtifact(
|
|
{
|
|
authorization: lookup,
|
|
metadata: new DrizzleGeneratedArtifactStore(),
|
|
storage: new LocalArtifactStorage(artifactRoot),
|
|
},
|
|
{
|
|
actor: { userId: viewerId },
|
|
workspaceId: isolatedWorkspaceId,
|
|
artifactId: generatedArtifactId,
|
|
},
|
|
),
|
|
).rejects.toMatchObject({ code: 'workspace_access_denied' })
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: editorId },
|
|
workspaceId: primaryWorkspaceId,
|
|
action: 'write',
|
|
}),
|
|
).resolves.toMatchObject({ workspaceRole: 'editor' })
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: editorId },
|
|
workspaceId: primaryWorkspaceId,
|
|
action: 'destructive',
|
|
}),
|
|
).rejects.toMatchObject({ code: 'workspace_access_denied' })
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: ownerId },
|
|
workspaceId: primaryWorkspaceId,
|
|
action: 'destructive',
|
|
}),
|
|
).resolves.toMatchObject({ workspaceRole: 'owner' })
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: viewerId },
|
|
workspaceId: isolatedWorkspaceId,
|
|
action: 'read',
|
|
}),
|
|
).rejects.toMatchObject({ code: 'workspace_access_denied' })
|
|
await expect(
|
|
authorizeWorkspaceAction(lookup, {
|
|
actor: { userId: administratorId },
|
|
workspaceId: primaryWorkspaceId,
|
|
action: 'read',
|
|
}),
|
|
).rejects.toMatchObject({ code: 'workspace_access_denied' })
|
|
})
|
|
|
|
it('persists the four representative golden compositions and their digests', async () => {
|
|
const slugs = [
|
|
'repository-health-audit',
|
|
'root-cause-bugfix',
|
|
'feature-from-spec',
|
|
'production-readiness-audit',
|
|
] as const
|
|
const records = await loadBuiltInPlaybookRecords()
|
|
const sql = getSqlClient()
|
|
const store = new DrizzleGeneratedRunStore()
|
|
const governedExampleProfile = await loadGovernedExampleProfile()
|
|
|
|
for (const slug of slugs) {
|
|
const record = records.find((candidate) => candidate.slug === slug)
|
|
expect(record, slug).toBeDefined()
|
|
const manifest = record!.packageJson as unknown as {
|
|
metadata: PlaybookMetadata
|
|
spec: PlaybookSpecification & {
|
|
compatibility?: { repositoryRequired?: boolean }
|
|
}
|
|
}
|
|
const example = parse(
|
|
await readFile(
|
|
path.resolve('content/playbooks', slug, 'examples/minimal.yaml'),
|
|
'utf8',
|
|
),
|
|
) as {
|
|
workMode: string
|
|
autonomyLevel: CanonicalPromptRequest['autonomyLevel']
|
|
inputs?: CanonicalPromptRequest['inputs'] & ImmutableJsonObject
|
|
repositoryProfile?: string
|
|
}
|
|
const [version] = await sql<{ id: string }[]>`
|
|
select pv.id
|
|
from playbook_versions pv
|
|
join playbooks p on p.id = pv.playbook_id
|
|
where p.slug = ${slug} and pv.semantic_version = ${record!.semanticVersion}
|
|
`
|
|
expect(version?.id, slug).toBeTruthy()
|
|
const inputs = example.inputs ?? {}
|
|
const selectedProfile =
|
|
example.repositoryProfile ||
|
|
manifest.spec.compatibility?.repositoryRequired
|
|
? exampleProfile
|
|
: null
|
|
const requiredInput = manifest.spec.inputs?.find(
|
|
(definition) => definition.required,
|
|
)
|
|
expect(requiredInput, `${slug} required input`).toBeDefined()
|
|
const invalidInputs = { ...inputs }
|
|
delete invalidInputs[requiredInput!.key]
|
|
await expect(
|
|
composeAndCreateGeneratedRun(
|
|
{
|
|
store,
|
|
nextId: randomUUID,
|
|
now: () => new Date(),
|
|
workspaceAuthorization: new DrizzleWorkspaceAuthorizationLookup(),
|
|
},
|
|
{
|
|
prompt: {
|
|
metadata: manifest.metadata,
|
|
specification: manifest.spec,
|
|
template: record!.templateText,
|
|
inputs: invalidInputs,
|
|
workMode: example.workMode,
|
|
autonomyLevel: example.autonomyLevel,
|
|
repositoryProfile: selectedProfile,
|
|
},
|
|
snapshots: {
|
|
playbook: record!.packageJson as unknown as ImmutableJsonObject,
|
|
repositoryProfile: selectedProfile
|
|
? governedExampleProfile
|
|
: null,
|
|
normalizedInput: invalidInputs,
|
|
policy: {
|
|
autonomyLevel: example.autonomyLevel,
|
|
conditionsResolved: true,
|
|
},
|
|
provenance: [],
|
|
},
|
|
lint: { exportReadiness: 'ready', findings: [] },
|
|
generatedBy: ownerId,
|
|
workspaceId: primaryWorkspaceId,
|
|
playbookVersionId: version!.id,
|
|
idempotencyKey: `milestone-zero-invalid-${slug}`,
|
|
},
|
|
),
|
|
).rejects.toMatchObject({ code: 'composition_input_invalid' })
|
|
const golden = await readFile(
|
|
path.resolve('examples/rendered-prompts', `${slug}.md`),
|
|
'utf8',
|
|
)
|
|
const result = await composeAndCreateGeneratedRun(
|
|
{
|
|
store,
|
|
nextId: randomUUID,
|
|
now: () => new Date(),
|
|
workspaceAuthorization: new DrizzleWorkspaceAuthorizationLookup(),
|
|
},
|
|
{
|
|
prompt: {
|
|
metadata: manifest.metadata,
|
|
specification: manifest.spec,
|
|
template: record!.templateText,
|
|
inputs,
|
|
workMode: example.workMode,
|
|
autonomyLevel: example.autonomyLevel,
|
|
repositoryProfile: selectedProfile,
|
|
},
|
|
snapshots: {
|
|
playbook: record!.packageJson as unknown as ImmutableJsonObject,
|
|
repositoryProfile: selectedProfile ? governedExampleProfile : null,
|
|
normalizedInput: inputs,
|
|
policy: {
|
|
autonomyLevel: example.autonomyLevel,
|
|
conditionsResolved: true,
|
|
},
|
|
provenance: [],
|
|
},
|
|
lint: { exportReadiness: 'ready', findings: [] },
|
|
generatedBy: ownerId,
|
|
workspaceId: primaryWorkspaceId,
|
|
playbookVersionId: version!.id,
|
|
idempotencyKey: `milestone-zero-representative-${slug}`,
|
|
},
|
|
)
|
|
|
|
expect(result.run.renderedPrompt, slug).toBe(golden)
|
|
expect(result.run.renderDigest, slug).toBe(sha256(golden))
|
|
const [persisted] = await sql<
|
|
{ renderedPrompt: string; renderDigest: string }[]
|
|
>`
|
|
select rendered_prompt as "renderedPrompt", render_digest as "renderDigest"
|
|
from generated_runs
|
|
where id = ${result.run.id} and workspace_id = ${primaryWorkspaceId}
|
|
`
|
|
expect(persisted, slug).toEqual({
|
|
renderedPrompt: golden,
|
|
renderDigest: sha256(golden),
|
|
})
|
|
}
|
|
}, 30_000)
|
|
})
|