This commit is contained in:
DevRunbook release export
@@ -0,0 +1,219 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { cpus, freemem, platform, release, totalmem } from 'node:os'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
|
||||
import {
|
||||
closeDatabase,
|
||||
DrizzlePlaybookCatalog,
|
||||
getSqlClient,
|
||||
} from '../../packages/db/src/index'
|
||||
import {
|
||||
assertBenchmarkDatabase,
|
||||
percentile,
|
||||
} from '../../packages/db/src/release/performance-benchmark'
|
||||
|
||||
const IDENTITY_COUNT = 1_000
|
||||
const VERSIONS_PER_IDENTITY = 10
|
||||
const VERSION_COUNT = IDENTITY_COUNT * VERSIONS_PER_IDENTITY
|
||||
const argumentsSet = new Set(process.argv.slice(2))
|
||||
const seed =
|
||||
argumentsSet.has('--seed') || argumentsSet.has('--seed-and-benchmark')
|
||||
const benchmark =
|
||||
argumentsSet.has('--benchmark') || argumentsSet.has('--seed-and-benchmark')
|
||||
const iterationsArgument = process.argv.find((value) =>
|
||||
value.startsWith('--iterations='),
|
||||
)
|
||||
const iterations = Number(iterationsArgument?.split('=')[1] ?? 100)
|
||||
|
||||
if ((!seed && !benchmark) || !Number.isInteger(iterations) || iterations < 30) {
|
||||
throw new Error(
|
||||
'Usage: performance-benchmark.mts (--seed|--benchmark|--seed-and-benchmark) [--iterations=100]; iterations must be at least 30.',
|
||||
)
|
||||
}
|
||||
|
||||
const sql = getSqlClient()
|
||||
const [database] = await sql<{ name: string; version: string }[]>`
|
||||
select current_database() as name, version() as version
|
||||
`
|
||||
if (!database) throw new Error('Unable to identify the benchmark database.')
|
||||
assertBenchmarkDatabase(database.name, process.env.DEVRUNBOOK_PERFORMANCE_ACK)
|
||||
|
||||
const [instance] = await sql<
|
||||
{ setup_completed_at: Date | null; owner_user_id: string | null }[]
|
||||
>`select setup_completed_at, owner_user_id from instance_settings where singleton`
|
||||
if (instance?.setup_completed_at || instance?.owner_user_id) {
|
||||
throw new Error('Refusing to seed or benchmark an initialized instance.')
|
||||
}
|
||||
|
||||
async function seedDataset(): Promise<void> {
|
||||
const [existing] = await sql<{ count: number }[]>`
|
||||
select count(*)::int as count from playbooks
|
||||
where namespace = 'performance-fixture'
|
||||
`
|
||||
if ((existing?.count ?? 0) !== 0) {
|
||||
throw new Error(
|
||||
'The deterministic performance fixture already exists; use --benchmark only.',
|
||||
)
|
||||
}
|
||||
await sql.begin(async (transaction) => {
|
||||
await transaction`
|
||||
insert into playbooks (
|
||||
id, workspace_id, logical_id, slug, namespace, source_type,
|
||||
created_at, updated_at
|
||||
)
|
||||
select
|
||||
md5('devrunbook-performance-playbook-' || identity)::uuid,
|
||||
null,
|
||||
'performance-fixture-' || lpad(identity::text, 4, '0'),
|
||||
'performance-fixture-' || lpad(identity::text, 4, '0'),
|
||||
'performance-fixture',
|
||||
'built_in',
|
||||
timestamptz '2026-01-01 00:00:00+00',
|
||||
timestamptz '2026-01-01 00:00:00+00'
|
||||
from generate_series(1, ${IDENTITY_COUNT}) identity
|
||||
`
|
||||
await transaction`
|
||||
insert into playbook_versions (
|
||||
id, playbook_id, semantic_version, lifecycle, package_api_version,
|
||||
title, summary, category, risk_tier, package_json, template_text,
|
||||
content_digest, search_document, published_at, created_at
|
||||
)
|
||||
select
|
||||
md5('devrunbook-performance-version-' || identity || '-' || version)::uuid,
|
||||
md5('devrunbook-performance-playbook-' || identity)::uuid,
|
||||
version::text || '.0.0',
|
||||
'reviewed',
|
||||
'devrunbook.io/v1.2',
|
||||
case identity % 4
|
||||
when 0 then 'Database migration performance fixture ' || identity
|
||||
when 1 then 'Frontend accessibility performance fixture ' || identity
|
||||
when 2 then 'Security review performance fixture ' || identity
|
||||
else 'Release operations performance fixture ' || identity
|
||||
end,
|
||||
'Deterministic indexed playbook version ' || version || ' for identity ' || identity || '.',
|
||||
case identity % 4
|
||||
when 0 then 'data-databases'
|
||||
when 1 then 'frontend-experience'
|
||||
when 2 then 'security-compliance'
|
||||
else 'release-operations'
|
||||
end,
|
||||
case identity % 3 when 0 then 'moderate' when 1 then 'high' else 'low' end,
|
||||
jsonb_build_object(
|
||||
'apiVersion', 'devrunbook.io/v1.2',
|
||||
'kind', 'Playbook',
|
||||
'metadata', jsonb_build_object(
|
||||
'id', 'performance-fixture-' || lpad(identity::text, 4, '0'),
|
||||
'slug', 'performance-fixture-' || lpad(identity::text, 4, '0'),
|
||||
'version', version::text || '.0.0',
|
||||
'tags', jsonb_build_array('performance', 'fixture',
|
||||
case identity % 4 when 0 then 'database' when 1 then 'accessibility' when 2 then 'security' else 'release' end)
|
||||
),
|
||||
'spec', jsonb_build_object(
|
||||
'type', 'guided',
|
||||
'modes', jsonb_build_array('inspect', 'plan'),
|
||||
'defaultMode', 'plan',
|
||||
'autonomy', jsonb_build_object('min', 'observe', 'max', 'verify', 'default', 'plan'),
|
||||
'compatibility', jsonb_build_object('languages', jsonb_build_array('TypeScript')),
|
||||
'intent', jsonb_build_object('problem', 'performance fixture', 'outcome', 'measured result')
|
||||
),
|
||||
'quality', jsonb_build_object('reviewStatus', 'technical-reviewed')
|
||||
),
|
||||
'# Performance fixture\n\nThis deterministic template is data and is never executed.\n',
|
||||
encode(digest('performance-fixture-' || identity || '-' || version, 'sha256'), 'hex'),
|
||||
to_tsvector('simple',
|
||||
case identity % 4
|
||||
when 0 then 'database migration performance fixture'
|
||||
when 1 then 'frontend accessibility performance fixture'
|
||||
when 2 then 'security review performance fixture'
|
||||
else 'release operations performance fixture'
|
||||
end || ' deterministic indexed playbook'),
|
||||
timestamptz '2026-01-01 00:00:00+00' + (version * interval '1 day'),
|
||||
timestamptz '2026-01-01 00:00:00+00'
|
||||
from generate_series(1, ${IDENTITY_COUNT}) identity
|
||||
cross join generate_series(1, ${VERSIONS_PER_IDENTITY}) version
|
||||
`
|
||||
})
|
||||
}
|
||||
|
||||
async function measure(): Promise<Readonly<Record<string, unknown>>> {
|
||||
const [dataset] = await sql<{ identities: number; versions: number }[]>`
|
||||
select count(distinct p.id)::int as identities, count(v.id)::int as versions
|
||||
from playbooks p join playbook_versions v on v.playbook_id = p.id
|
||||
where p.namespace = 'performance-fixture'
|
||||
`
|
||||
if (
|
||||
dataset?.identities !== IDENTITY_COUNT ||
|
||||
dataset.versions !== VERSION_COUNT
|
||||
) {
|
||||
throw new Error(
|
||||
`Expected ${IDENTITY_COUNT} identities and ${VERSION_COUNT} versions.`,
|
||||
)
|
||||
}
|
||||
const catalog = new DrizzlePlaybookCatalog()
|
||||
const terms = ['database migration', 'accessibility', 'security', 'release']
|
||||
for (let index = 0; index < 10; index += 1) {
|
||||
await catalog.list({ q: terms[index % terms.length] })
|
||||
}
|
||||
const searchSamples: number[] = []
|
||||
const detailSamples: number[] = []
|
||||
for (let index = 0; index < iterations; index += 1) {
|
||||
const searchStart = performance.now()
|
||||
await catalog.list({ q: terms[index % terms.length] })
|
||||
searchSamples.push(performance.now() - searchStart)
|
||||
const detailStart = performance.now()
|
||||
await catalog.findBySlug(
|
||||
`performance-fixture-${String((index % IDENTITY_COUNT) + 1).padStart(4, '0')}`,
|
||||
'built_in',
|
||||
)
|
||||
detailSamples.push(performance.now() - detailStart)
|
||||
}
|
||||
const metrics = (samples: readonly number[], targetMs: number) => ({
|
||||
samples: samples.length,
|
||||
p50Ms: Number(percentile(samples, 0.5).toFixed(3)),
|
||||
p95Ms: Number(percentile(samples, 0.95).toFixed(3)),
|
||||
p99Ms: Number(percentile(samples, 0.99).toFixed(3)),
|
||||
targetMs,
|
||||
meetsReferenceTarget: percentile(samples, 0.95) < targetMs,
|
||||
})
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
fixture: {
|
||||
identityCount: dataset.identities,
|
||||
versionsPerIdentity: VERSIONS_PER_IDENTITY,
|
||||
versionCount: dataset.versions,
|
||||
digest: createHash('sha256')
|
||||
.update(
|
||||
`devrunbook-performance-v1:${IDENTITY_COUNT}:${VERSIONS_PER_IDENTITY}`,
|
||||
)
|
||||
.digest('hex'),
|
||||
},
|
||||
environment: {
|
||||
applicationCommit: process.env.DEVRUNBOOK_APPLICATION_COMMIT ?? null,
|
||||
databaseName: database.name,
|
||||
databaseVersion: database.version,
|
||||
nodeVersion: process.version,
|
||||
platform: `${platform()} ${release()}`,
|
||||
cpuModel: cpus()[0]?.model ?? 'unknown',
|
||||
cpuCount: cpus().length,
|
||||
totalMemoryBytes: totalmem(),
|
||||
freeMemoryBytesAtCompletion: freemem(),
|
||||
},
|
||||
method: { warmupIterations: 10, measuredIterations: iterations },
|
||||
metrics: {
|
||||
librarySearch: metrics(searchSamples, 500),
|
||||
playbookDetail: metrics(detailSamples, 400),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (seed) await seedDataset()
|
||||
if (benchmark)
|
||||
process.stdout.write(`${JSON.stringify(await measure(), null, 2)}\n`)
|
||||
else
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ seededIdentities: IDENTITY_COUNT, seededVersions: VERSION_COUNT })}\n`,
|
||||
)
|
||||
} finally {
|
||||
await closeDatabase()
|
||||
}
|
||||
Reference in New Issue
Block a user