220 lines
8.7 KiB
TypeScript
220 lines
8.7 KiB
TypeScript
import { createHash } from 'node:crypto'
|
|
import { cpus, freemem, platform, release, totalmem } from 'node:os'
|
|
import { performance } from 'node:perf_hooks'
|
|
|
|
import {
|
|
closeDatabase,
|
|
DrizzlePlaybookCatalog,
|
|
getSqlClient,
|
|
} from '../../packages/db/src/index'
|
|
import {
|
|
assertBenchmarkDatabase,
|
|
percentile,
|
|
} from '../../packages/db/src/release/performance-benchmark'
|
|
|
|
const IDENTITY_COUNT = 1_000
|
|
const VERSIONS_PER_IDENTITY = 10
|
|
const VERSION_COUNT = IDENTITY_COUNT * VERSIONS_PER_IDENTITY
|
|
const argumentsSet = new Set(process.argv.slice(2))
|
|
const seed =
|
|
argumentsSet.has('--seed') || argumentsSet.has('--seed-and-benchmark')
|
|
const benchmark =
|
|
argumentsSet.has('--benchmark') || argumentsSet.has('--seed-and-benchmark')
|
|
const iterationsArgument = process.argv.find((value) =>
|
|
value.startsWith('--iterations='),
|
|
)
|
|
const iterations = Number(iterationsArgument?.split('=')[1] ?? 100)
|
|
|
|
if ((!seed && !benchmark) || !Number.isInteger(iterations) || iterations < 30) {
|
|
throw new Error(
|
|
'Usage: performance-benchmark.mts (--seed|--benchmark|--seed-and-benchmark) [--iterations=100]; iterations must be at least 30.',
|
|
)
|
|
}
|
|
|
|
const sql = getSqlClient()
|
|
const [database] = await sql<{ name: string; version: string }[]>`
|
|
select current_database() as name, version() as version
|
|
`
|
|
if (!database) throw new Error('Unable to identify the benchmark database.')
|
|
assertBenchmarkDatabase(database.name, process.env.DEVRUNBOOK_PERFORMANCE_ACK)
|
|
|
|
const [instance] = await sql<
|
|
{ setup_completed_at: Date | null; owner_user_id: string | null }[]
|
|
>`select setup_completed_at, owner_user_id from instance_settings where singleton`
|
|
if (instance?.setup_completed_at || instance?.owner_user_id) {
|
|
throw new Error('Refusing to seed or benchmark an initialized instance.')
|
|
}
|
|
|
|
async function seedDataset(): Promise<void> {
|
|
const [existing] = await sql<{ count: number }[]>`
|
|
select count(*)::int as count from playbooks
|
|
where namespace = 'performance-fixture'
|
|
`
|
|
if ((existing?.count ?? 0) !== 0) {
|
|
throw new Error(
|
|
'The deterministic performance fixture already exists; use --benchmark only.',
|
|
)
|
|
}
|
|
await sql.begin(async (transaction) => {
|
|
await transaction`
|
|
insert into playbooks (
|
|
id, workspace_id, logical_id, slug, namespace, source_type,
|
|
created_at, updated_at
|
|
)
|
|
select
|
|
md5('devrunbook-performance-playbook-' || identity)::uuid,
|
|
null,
|
|
'performance-fixture-' || lpad(identity::text, 4, '0'),
|
|
'performance-fixture-' || lpad(identity::text, 4, '0'),
|
|
'performance-fixture',
|
|
'built_in',
|
|
timestamptz '2026-01-01 00:00:00+00',
|
|
timestamptz '2026-01-01 00:00:00+00'
|
|
from generate_series(1, ${IDENTITY_COUNT}) identity
|
|
`
|
|
await transaction`
|
|
insert into playbook_versions (
|
|
id, playbook_id, semantic_version, lifecycle, package_api_version,
|
|
title, summary, category, risk_tier, package_json, template_text,
|
|
content_digest, search_document, published_at, created_at
|
|
)
|
|
select
|
|
md5('devrunbook-performance-version-' || identity || '-' || version)::uuid,
|
|
md5('devrunbook-performance-playbook-' || identity)::uuid,
|
|
version::text || '.0.0',
|
|
'reviewed',
|
|
'devrunbook.io/v1.2',
|
|
case identity % 4
|
|
when 0 then 'Database migration performance fixture ' || identity
|
|
when 1 then 'Frontend accessibility performance fixture ' || identity
|
|
when 2 then 'Security review performance fixture ' || identity
|
|
else 'Release operations performance fixture ' || identity
|
|
end,
|
|
'Deterministic indexed playbook version ' || version || ' for identity ' || identity || '.',
|
|
case identity % 4
|
|
when 0 then 'data-databases'
|
|
when 1 then 'frontend-experience'
|
|
when 2 then 'security-compliance'
|
|
else 'release-operations'
|
|
end,
|
|
case identity % 3 when 0 then 'moderate' when 1 then 'high' else 'low' end,
|
|
jsonb_build_object(
|
|
'apiVersion', 'devrunbook.io/v1.2',
|
|
'kind', 'Playbook',
|
|
'metadata', jsonb_build_object(
|
|
'id', 'performance-fixture-' || lpad(identity::text, 4, '0'),
|
|
'slug', 'performance-fixture-' || lpad(identity::text, 4, '0'),
|
|
'version', version::text || '.0.0',
|
|
'tags', jsonb_build_array('performance', 'fixture',
|
|
case identity % 4 when 0 then 'database' when 1 then 'accessibility' when 2 then 'security' else 'release' end)
|
|
),
|
|
'spec', jsonb_build_object(
|
|
'type', 'guided',
|
|
'modes', jsonb_build_array('inspect', 'plan'),
|
|
'defaultMode', 'plan',
|
|
'autonomy', jsonb_build_object('min', 'observe', 'max', 'verify', 'default', 'plan'),
|
|
'compatibility', jsonb_build_object('languages', jsonb_build_array('TypeScript')),
|
|
'intent', jsonb_build_object('problem', 'performance fixture', 'outcome', 'measured result')
|
|
),
|
|
'quality', jsonb_build_object('reviewStatus', 'technical-reviewed')
|
|
),
|
|
'# Performance fixture\n\nThis deterministic template is data and is never executed.\n',
|
|
encode(digest('performance-fixture-' || identity || '-' || version, 'sha256'), 'hex'),
|
|
to_tsvector('simple',
|
|
case identity % 4
|
|
when 0 then 'database migration performance fixture'
|
|
when 1 then 'frontend accessibility performance fixture'
|
|
when 2 then 'security review performance fixture'
|
|
else 'release operations performance fixture'
|
|
end || ' deterministic indexed playbook'),
|
|
timestamptz '2026-01-01 00:00:00+00' + (version * interval '1 day'),
|
|
timestamptz '2026-01-01 00:00:00+00'
|
|
from generate_series(1, ${IDENTITY_COUNT}) identity
|
|
cross join generate_series(1, ${VERSIONS_PER_IDENTITY}) version
|
|
`
|
|
})
|
|
}
|
|
|
|
async function measure(): Promise<Readonly<Record<string, unknown>>> {
|
|
const [dataset] = await sql<{ identities: number; versions: number }[]>`
|
|
select count(distinct p.id)::int as identities, count(v.id)::int as versions
|
|
from playbooks p join playbook_versions v on v.playbook_id = p.id
|
|
where p.namespace = 'performance-fixture'
|
|
`
|
|
if (
|
|
dataset?.identities !== IDENTITY_COUNT ||
|
|
dataset.versions !== VERSION_COUNT
|
|
) {
|
|
throw new Error(
|
|
`Expected ${IDENTITY_COUNT} identities and ${VERSION_COUNT} versions.`,
|
|
)
|
|
}
|
|
const catalog = new DrizzlePlaybookCatalog()
|
|
const terms = ['database migration', 'accessibility', 'security', 'release']
|
|
for (let index = 0; index < 10; index += 1) {
|
|
await catalog.list({ q: terms[index % terms.length] })
|
|
}
|
|
const searchSamples: number[] = []
|
|
const detailSamples: number[] = []
|
|
for (let index = 0; index < iterations; index += 1) {
|
|
const searchStart = performance.now()
|
|
await catalog.list({ q: terms[index % terms.length] })
|
|
searchSamples.push(performance.now() - searchStart)
|
|
const detailStart = performance.now()
|
|
await catalog.findBySlug(
|
|
`performance-fixture-${String((index % IDENTITY_COUNT) + 1).padStart(4, '0')}`,
|
|
'built_in',
|
|
)
|
|
detailSamples.push(performance.now() - detailStart)
|
|
}
|
|
const metrics = (samples: readonly number[], targetMs: number) => ({
|
|
samples: samples.length,
|
|
p50Ms: Number(percentile(samples, 0.5).toFixed(3)),
|
|
p95Ms: Number(percentile(samples, 0.95).toFixed(3)),
|
|
p99Ms: Number(percentile(samples, 0.99).toFixed(3)),
|
|
targetMs,
|
|
meetsReferenceTarget: percentile(samples, 0.95) < targetMs,
|
|
})
|
|
return {
|
|
schemaVersion: 1,
|
|
fixture: {
|
|
identityCount: dataset.identities,
|
|
versionsPerIdentity: VERSIONS_PER_IDENTITY,
|
|
versionCount: dataset.versions,
|
|
digest: createHash('sha256')
|
|
.update(
|
|
`devrunbook-performance-v1:${IDENTITY_COUNT}:${VERSIONS_PER_IDENTITY}`,
|
|
)
|
|
.digest('hex'),
|
|
},
|
|
environment: {
|
|
applicationCommit: process.env.DEVRUNBOOK_APPLICATION_COMMIT ?? null,
|
|
databaseName: database.name,
|
|
databaseVersion: database.version,
|
|
nodeVersion: process.version,
|
|
platform: `${platform()} ${release()}`,
|
|
cpuModel: cpus()[0]?.model ?? 'unknown',
|
|
cpuCount: cpus().length,
|
|
totalMemoryBytes: totalmem(),
|
|
freeMemoryBytesAtCompletion: freemem(),
|
|
},
|
|
method: { warmupIterations: 10, measuredIterations: iterations },
|
|
metrics: {
|
|
librarySearch: metrics(searchSamples, 500),
|
|
playbookDetail: metrics(detailSamples, 400),
|
|
},
|
|
}
|
|
}
|
|
|
|
try {
|
|
if (seed) await seedDataset()
|
|
if (benchmark)
|
|
process.stdout.write(`${JSON.stringify(await measure(), null, 2)}\n`)
|
|
else
|
|
process.stdout.write(
|
|
`${JSON.stringify({ seededIdentities: IDENTITY_COUNT, seededVersions: VERSION_COUNT })}\n`,
|
|
)
|
|
} finally {
|
|
await closeDatabase()
|
|
}
|