import { createHash } from 'node:crypto' import { cpus, freemem, platform, release, totalmem } from 'node:os' import { performance } from 'node:perf_hooks' import { closeDatabase, DrizzlePlaybookCatalog, getSqlClient, } from '../../packages/db/src/index' import { assertBenchmarkDatabase, percentile, } from '../../packages/db/src/release/performance-benchmark' const IDENTITY_COUNT = 1_000 const VERSIONS_PER_IDENTITY = 10 const VERSION_COUNT = IDENTITY_COUNT * VERSIONS_PER_IDENTITY const argumentsSet = new Set(process.argv.slice(2)) const seed = argumentsSet.has('--seed') || argumentsSet.has('--seed-and-benchmark') const benchmark = argumentsSet.has('--benchmark') || argumentsSet.has('--seed-and-benchmark') const iterationsArgument = process.argv.find((value) => value.startsWith('--iterations='), ) const iterations = Number(iterationsArgument?.split('=')[1] ?? 100) if ((!seed && !benchmark) || !Number.isInteger(iterations) || iterations < 30) { throw new Error( 'Usage: performance-benchmark.mts (--seed|--benchmark|--seed-and-benchmark) [--iterations=100]; iterations must be at least 30.', ) } const sql = getSqlClient() const [database] = await sql<{ name: string; version: string }[]>` select current_database() as name, version() as version ` if (!database) throw new Error('Unable to identify the benchmark database.') assertBenchmarkDatabase(database.name, process.env.DEVRUNBOOK_PERFORMANCE_ACK) const [instance] = await sql< { setup_completed_at: Date | null; owner_user_id: string | null }[] >`select setup_completed_at, owner_user_id from instance_settings where singleton` if (instance?.setup_completed_at || instance?.owner_user_id) { throw new Error('Refusing to seed or benchmark an initialized instance.') } async function seedDataset(): Promise { const [existing] = await sql<{ count: number }[]>` select count(*)::int as count from playbooks where namespace = 'performance-fixture' ` if ((existing?.count ?? 0) !== 0) { throw new Error( 'The deterministic performance fixture already exists; use --benchmark only.', ) } await sql.begin(async (transaction) => { await transaction` insert into playbooks ( id, workspace_id, logical_id, slug, namespace, source_type, created_at, updated_at ) select md5('devrunbook-performance-playbook-' || identity)::uuid, null, 'performance-fixture-' || lpad(identity::text, 4, '0'), 'performance-fixture-' || lpad(identity::text, 4, '0'), 'performance-fixture', 'built_in', timestamptz '2026-01-01 00:00:00+00', timestamptz '2026-01-01 00:00:00+00' from generate_series(1, ${IDENTITY_COUNT}) identity ` await transaction` insert into playbook_versions ( id, playbook_id, semantic_version, lifecycle, package_api_version, title, summary, category, risk_tier, package_json, template_text, content_digest, search_document, published_at, created_at ) select md5('devrunbook-performance-version-' || identity || '-' || version)::uuid, md5('devrunbook-performance-playbook-' || identity)::uuid, version::text || '.0.0', 'reviewed', 'devrunbook.io/v1.2', case identity % 4 when 0 then 'Database migration performance fixture ' || identity when 1 then 'Frontend accessibility performance fixture ' || identity when 2 then 'Security review performance fixture ' || identity else 'Release operations performance fixture ' || identity end, 'Deterministic indexed playbook version ' || version || ' for identity ' || identity || '.', case identity % 4 when 0 then 'data-databases' when 1 then 'frontend-experience' when 2 then 'security-compliance' else 'release-operations' end, case identity % 3 when 0 then 'moderate' when 1 then 'high' else 'low' end, jsonb_build_object( 'apiVersion', 'devrunbook.io/v1.2', 'kind', 'Playbook', 'metadata', jsonb_build_object( 'id', 'performance-fixture-' || lpad(identity::text, 4, '0'), 'slug', 'performance-fixture-' || lpad(identity::text, 4, '0'), 'version', version::text || '.0.0', 'tags', jsonb_build_array('performance', 'fixture', case identity % 4 when 0 then 'database' when 1 then 'accessibility' when 2 then 'security' else 'release' end) ), 'spec', jsonb_build_object( 'type', 'guided', 'modes', jsonb_build_array('inspect', 'plan'), 'defaultMode', 'plan', 'autonomy', jsonb_build_object('min', 'observe', 'max', 'verify', 'default', 'plan'), 'compatibility', jsonb_build_object('languages', jsonb_build_array('TypeScript')), 'intent', jsonb_build_object('problem', 'performance fixture', 'outcome', 'measured result') ), 'quality', jsonb_build_object('reviewStatus', 'technical-reviewed') ), '# Performance fixture\n\nThis deterministic template is data and is never executed.\n', encode(digest('performance-fixture-' || identity || '-' || version, 'sha256'), 'hex'), to_tsvector('simple', case identity % 4 when 0 then 'database migration performance fixture' when 1 then 'frontend accessibility performance fixture' when 2 then 'security review performance fixture' else 'release operations performance fixture' end || ' deterministic indexed playbook'), timestamptz '2026-01-01 00:00:00+00' + (version * interval '1 day'), timestamptz '2026-01-01 00:00:00+00' from generate_series(1, ${IDENTITY_COUNT}) identity cross join generate_series(1, ${VERSIONS_PER_IDENTITY}) version ` }) } async function measure(): Promise>> { const [dataset] = await sql<{ identities: number; versions: number }[]>` select count(distinct p.id)::int as identities, count(v.id)::int as versions from playbooks p join playbook_versions v on v.playbook_id = p.id where p.namespace = 'performance-fixture' ` if ( dataset?.identities !== IDENTITY_COUNT || dataset.versions !== VERSION_COUNT ) { throw new Error( `Expected ${IDENTITY_COUNT} identities and ${VERSION_COUNT} versions.`, ) } const catalog = new DrizzlePlaybookCatalog() const terms = ['database migration', 'accessibility', 'security', 'release'] for (let index = 0; index < 10; index += 1) { await catalog.list({ q: terms[index % terms.length] }) } const searchSamples: number[] = [] const detailSamples: number[] = [] for (let index = 0; index < iterations; index += 1) { const searchStart = performance.now() await catalog.list({ q: terms[index % terms.length] }) searchSamples.push(performance.now() - searchStart) const detailStart = performance.now() await catalog.findBySlug( `performance-fixture-${String((index % IDENTITY_COUNT) + 1).padStart(4, '0')}`, 'built_in', ) detailSamples.push(performance.now() - detailStart) } const metrics = (samples: readonly number[], targetMs: number) => ({ samples: samples.length, p50Ms: Number(percentile(samples, 0.5).toFixed(3)), p95Ms: Number(percentile(samples, 0.95).toFixed(3)), p99Ms: Number(percentile(samples, 0.99).toFixed(3)), targetMs, meetsReferenceTarget: percentile(samples, 0.95) < targetMs, }) return { schemaVersion: 1, fixture: { identityCount: dataset.identities, versionsPerIdentity: VERSIONS_PER_IDENTITY, versionCount: dataset.versions, digest: createHash('sha256') .update( `devrunbook-performance-v1:${IDENTITY_COUNT}:${VERSIONS_PER_IDENTITY}`, ) .digest('hex'), }, environment: { applicationCommit: process.env.DEVRUNBOOK_APPLICATION_COMMIT ?? null, databaseName: database.name, databaseVersion: database.version, nodeVersion: process.version, platform: `${platform()} ${release()}`, cpuModel: cpus()[0]?.model ?? 'unknown', cpuCount: cpus().length, totalMemoryBytes: totalmem(), freeMemoryBytesAtCompletion: freemem(), }, method: { warmupIterations: 10, measuredIterations: iterations }, metrics: { librarySearch: metrics(searchSamples, 500), playbookDetail: metrics(detailSamples, 400), }, } } try { if (seed) await seedDataset() if (benchmark) process.stdout.write(`${JSON.stringify(await measure(), null, 2)}\n`) else process.stdout.write( `${JSON.stringify({ seededIdentities: IDENTITY_COUNT, seededVersions: VERSION_COUNT })}\n`, ) } finally { await closeDatabase() }