diff --git a/conformance/package.json b/conformance/package.json index 0a44c965d..9b96f3898 100644 --- a/conformance/package.json +++ b/conformance/package.json @@ -12,6 +12,7 @@ "test:parser-only": "tsx runner/index.ts --parser-only", "update-golden": "tsx runner/index.ts --update-golden", "script-size-check": "tsx runner/script-size-check.ts", + "script-metrics": "tsx runner/script-metrics.ts", "fuzz": "tsx fuzzer/index.ts", "fuzz:quick": "tsx fuzzer/index.ts --ir --compilers ts,go,rust,python,zig,ruby,java --render native --num 20", "fuzz:property": "tsx fuzzer/index.ts --property", diff --git a/conformance/runner/__tests__/script-metrics.test.ts b/conformance/runner/__tests__/script-metrics.test.ts new file mode 100644 index 000000000..ff9544176 --- /dev/null +++ b/conformance/runner/__tests__/script-metrics.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + parseArgs, + formatTable, + formatComparison, + formatDetail, + measureGolden, + tsSourcePath, + VARIANTS, + type FixtureMetrics, +} from '../script-metrics.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function fakeMetrics(fixture: string, scriptBytes: number): FixtureMetrics { + return { + fixture, + source: 'compiled', + scriptBytes, + opcodeCount: scriptBytes, + pushCount: 0, + categories: { + 'const-push': scriptBytes, 'small-int-push': 0, 'stack-shuffle': 0, + arithmetic: 0, bytes: 0, crypto: 0, control: 0, other: 0, + }, + opcodes: { PUSH: 1 }, + constants: [], + }; +} + +// --------------------------------------------------------------------------- + +describe('parseArgs', () => { + it('defaults to reading goldens with a summary table', () => { + const a = parseArgs([]); + expect(a.compileMode).toBe(false); + expect(a.detail).toBe(false); + expect(a.compare).toEqual([]); + }); + + it('--compare implies --compile', () => { + const a = parseArgs(['--compare', 'current,current']); + expect(a.compare).toEqual(['current', 'current']); + expect(a.compileMode).toBe(true); + }); + + it('rejects an unknown argument instead of ignoring it', () => { + // A silently-ignored flag in a benchmark harness reads as "I measured + // that" when nothing was measured. + expect(() => parseArgs(['--nope'])).toThrow(/unknown argument/); + }); +}); + +describe('variants', () => { + it('always offers the shipping default as the comparison base', () => { + expect(VARIANTS.current).toBeDefined(); + expect(VARIANTS.current).toEqual({}); + }); +}); + +describe('tsSourcePath', () => { + it('resolves a fixture that ships a TypeScript source', () => { + const p = tsSourcePath('p2pkh') ?? tsSourcePath('basic-p2pkh'); + expect(p).toMatch(/\.runar\.ts$/); + }); + + it('returns null for a fixture that declares no .runar.ts rather than throwing', () => { + // A size report must skip such a fixture visibly, not crash on it. + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + mkdirSync(join(dir, 'go-only')); + writeFileSync( + join(dir, 'go-only', 'source.json'), + JSON.stringify({ sources: { '.runar.go': './X.runar.go' }, compilers: ['go'] }), + ); + expect(tsSourcePath('go-only', dir)).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('returns null when the fixture has no source.json at all', () => { + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + mkdirSync(join(dir, 'bare')); + expect(tsSourcePath('bare', dir)).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('throws when source.json points at a file that is not on disk', () => { + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + mkdirSync(join(dir, 'ghost')); + writeFileSync( + join(dir, 'ghost', 'source.json'), + JSON.stringify({ sources: { '.runar.ts': './nope.runar.ts' } }), + ); + expect(() => tsSourcePath('ghost', dir)).toThrow(/missing file/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('measureGolden', () => { + it('reports the byte length and category split of a golden', () => { + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + const hexPath = join(dir, 'expected-script.hex'); + // OP_DUP OP_HASH160 <20 bytes> OP_EQUALVERIFY OP_CHECKSIG + writeFileSync(hexPath, `76a914${'ab'.repeat(20)}88ac\n`); + const m = measureGolden('p2pkh-ish', hexPath); + expect(m.scriptBytes).toBe(25); + expect(m.source).toBe('golden'); + expect(m.categories['const-push']).toBe(21); + expect(m.opcodes['OP_CHECKSIG']).toBe(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('formatting', () => { + it('orders the summary table largest-first', () => { + const table = formatTable([fakeMetrics('small', 10), fakeMetrics('big', 100)]); + expect(table.indexOf('| big |')).toBeLessThan(table.indexOf('| small |')); + }); + + it('reports the delta of the last variant against the first', () => { + const byVariant = new Map([ + ['current', new Map([['fx', fakeMetrics('fx', 1000)]])], + ['tuned', new Map([['fx', fakeMetrics('fx', 250)]])], + ]); + const out = formatComparison(['current', 'tuned'], byVariant); + expect(out).toContain('-75.0%'); + }); + + it('renders a dash for a variant that produced no result', () => { + const byVariant = new Map([ + ['current', new Map([['fx', fakeMetrics('fx', 1000)]])], + ['tuned', new Map()], + ]); + expect(formatComparison(['current', 'tuned'], byVariant)).toContain('| — |'); + }); + + it('lists dominant constants in the detail view', () => { + const m = fakeMetrics('fx', 340); + m.constants = [{ hex: 'ff'.repeat(33), count: 10, bytes: 340 }]; + const out = formatDetail(m); + expect(out).toContain('33 B'); + expect(out).toContain('100.0%'); + }); +}); diff --git a/conformance/runner/script-metrics.ts b/conformance/runner/script-metrics.ts new file mode 100644 index 000000000..76acb6b81 --- /dev/null +++ b/conformance/runner/script-metrics.ts @@ -0,0 +1,317 @@ +/** + * Script-size instrumentation runner. + * + * `script-size-check.ts` answers "did any fixture grow?"; this answers the + * next question: "where did the bytes go?". For every conformance fixture it + * buckets the serialized script by byte category and reports the constants + * that dominate, so an optimization can be aimed at the term that actually + * costs something rather than at whatever is easiest to change. + * + * Two sources of script bytes: + * + * default — read the checked-in `expected-script.hex`. Fast, needs no + * compilation, and is exactly what CI ships. + * --compile — recompile each fixture's `.runar.ts` through the TS reference + * compiler with a given option set. This is how an experimental + * flag is benchmarked against the baseline; pass `--compare` to + * run several option sets and print the deltas. + * + * Usage: + * tsx runner/script-metrics.ts # goldens, markdown table + * tsx runner/script-metrics.ts --json out.json # machine-readable + * tsx runner/script-metrics.ts --fixture p256-wallet --detail + * tsx runner/script-metrics.ts --compile --compare current,liveness + * + * Read-only: it never writes a golden or a baseline. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { + analyzeScriptHex, + type ByteCategory, + type ScriptMetrics, +} from '../../packages/runar-compiler/src/metrics/script-metrics.js'; +import { compile, type CompileOptions } from '../../packages/runar-compiler/src/index.js'; +import { CONFORMANCE_ROOT, TESTS_DIR, discoverFixtures } from './script-size-check.js'; + +const REPO_ROOT = resolve(CONFORMANCE_ROOT, '..'); + +// --------------------------------------------------------------------------- +// Fixture sources +// --------------------------------------------------------------------------- + +interface SourceConfig { + sources?: Record; + compilers?: string[]; +} + +/** Absolute path to a fixture's `.runar.ts` source, or null if it ships none. */ +export function tsSourcePath(fixture: string, testsDir: string = TESTS_DIR): string | null { + const configFile = join(testsDir, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf8')) as SourceConfig; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(testsDir, fixture, rel); + if (!existsSync(abs)) { + throw new Error(`source.json for '${fixture}' points at a missing file: ${abs}`); + } + return abs; +} + +// --------------------------------------------------------------------------- +// Option sets ("variants") a benchmark can compare +// --------------------------------------------------------------------------- + +/** + * Named compiler configurations. `current` is the shipping default; every + * other entry is an experimental flag combination. Adding a variant here is + * all it takes to get it into the comparison table. + */ +export const VARIANTS: Record = { + current: {}, + 'ec-pool': { ecConstantPool: true }, + liveness: { schedulerMode: 'liveness' }, + both: { ecConstantPool: true, schedulerMode: 'liveness' }, +}; + +// --------------------------------------------------------------------------- +// Measurement +// --------------------------------------------------------------------------- + +export interface FixtureMetrics extends ScriptMetrics { + fixture: string; + /** Where the bytes came from: the checked-in golden or a live compile. */ + source: 'golden' | 'compiled'; +} + +export function measureGolden(fixture: string, hexPath: string): FixtureMetrics { + const hex = readFileSync(hexPath, 'utf8').replace(/\s+/g, ''); + return { fixture, source: 'golden', ...analyzeScriptHex(hex) }; +} + +export function measureCompiled( + fixture: string, + sourcePath: string, + options: CompileOptions, +): FixtureMetrics { + const source = readFileSync(sourcePath, 'utf8'); + const result = compile(source, { ...options, fileName: sourcePath }); + if (!result.success || result.scriptHex === undefined) { + const errs = result.diagnostics.filter(d => d.severity === 'error').map(d => d.message); + throw new Error(`compile failed for '${fixture}': ${errs.join('; ') || 'no scriptHex'}`); + } + return { fixture, source: 'compiled', ...analyzeScriptHex(result.scriptHex) }; +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +const REPORT_CATEGORIES: ByteCategory[] = [ + 'const-push', 'stack-shuffle', 'arithmetic', 'bytes', 'crypto', + 'small-int-push', 'control', 'other', +]; + +function pct(part: number, whole: number): string { + if (whole === 0) return '0.0%'; + return `${((100 * part) / whole).toFixed(1)}%`; +} + +export function formatTable(rows: FixtureMetrics[]): string { + const sorted = [...rows].sort((a, b) => b.scriptBytes - a.scriptBytes); + const head = ['fixture', 'bytes', 'ops', ...REPORT_CATEGORIES]; + const lines = [ + `| ${head.join(' | ')} |`, + `|${head.map((_, i) => (i === 0 ? '---' : '---:')).join('|')}|`, + ]; + for (const r of sorted) { + const cells = REPORT_CATEGORIES.map(c => { + const v = r.categories[c]; + return v === 0 ? '—' : `${v} (${pct(v, r.scriptBytes)})`; + }); + lines.push(`| ${r.fixture} | ${r.scriptBytes} | ${r.opcodeCount} | ${cells.join(' | ')} |`); + } + return lines.join('\n'); +} + +/** Per-fixture detail: the constants that dominate, and the opcode histogram. */ +export function formatDetail(m: FixtureMetrics, topN = 8): string { + const out: string[] = []; + out.push(`### ${m.fixture} — ${m.scriptBytes} bytes, ${m.opcodeCount} ops (${m.source})`); + out.push(''); + out.push('| category | bytes | share |'); + out.push('|---|---:|---:|'); + for (const c of REPORT_CATEGORIES) { + const v = m.categories[c]; + if (v > 0) out.push(`| ${c} | ${v} | ${pct(v, m.scriptBytes)} |`); + } + out.push(''); + if (m.constants.length > 0) { + out.push('| repeated constant | size | pushes | total bytes | share |'); + out.push('|---|---:|---:|---:|---:|'); + for (const c of m.constants.slice(0, topN)) { + const label = c.hex.length > 24 ? `${c.hex.slice(0, 20)}…` : c.hex; + out.push(`| \`${label}\` | ${c.hex.length / 2} B | ${c.count} | ${c.bytes} | ${pct(c.bytes, m.scriptBytes)} |`); + } + out.push(''); + } + const ops = Object.entries(m.opcodes).sort((a, b) => b[1] - a[1]).slice(0, topN); + out.push(`Top opcodes: ${ops.map(([k, v]) => `${k}×${v}`).join(', ')}`); + return out.join('\n'); +} + +/** Side-by-side variant comparison for one fixture set. */ +export function formatComparison( + variantNames: string[], + byVariant: Map>, +): string { + const base = variantNames[0]!; + const fixtures = [...(byVariant.get(base)?.keys() ?? [])] + .sort((a, b) => (byVariant.get(base)!.get(b)!.scriptBytes) - (byVariant.get(base)!.get(a)!.scriptBytes)); + const head = ['fixture', ...variantNames.map(v => (v === base ? `${v} (base)` : `${v}`)), 'delta']; + const lines = [ + `| ${head.join(' | ')} |`, + `|${head.map((_, i) => (i === 0 ? '---' : '---:')).join('|')}|`, + ]; + for (const fx of fixtures) { + const baseBytes = byVariant.get(base)!.get(fx)!.scriptBytes; + const cells = variantNames.map(v => { + const m = byVariant.get(v)?.get(fx); + return m ? String(m.scriptBytes) : '—'; + }); + const last = byVariant.get(variantNames[variantNames.length - 1]!)?.get(fx); + const delta = last && baseBytes > 0 + ? `${(((last.scriptBytes - baseBytes) / baseBytes) * 100).toFixed(1)}%` + : '—'; + lines.push(`| ${fx} | ${cells.join(' | ')} | ${delta} |`); + } + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +interface Args { + json?: string; + fixture?: string; + detail: boolean; + compileMode: boolean; + compare: string[]; + top: number; +} + +export function parseArgs(argv: string[]): Args { + const args: Args = { detail: false, compileMode: false, compare: [], top: 8 }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + if (a === '--json') args.json = argv[++i]; + else if (a === '--fixture') args.fixture = argv[++i]; + else if (a === '--detail') args.detail = true; + else if (a === '--compile') args.compileMode = true; + else if (a === '--compare') { args.compare = (argv[++i] ?? '').split(',').filter(Boolean); args.compileMode = true; } + else if (a === '--top') args.top = Number(argv[++i]); + else if (a === '--help' || a === '-h') { args.detail = false; printHelp(); process.exit(0); } + else throw new Error(`script-metrics: unknown argument '${a}'`); + } + return args; +} + +function printHelp(): void { + console.log(` +script-metrics — where do a fixture's script bytes go? + + --fixture measure one fixture instead of all + --detail per-fixture category / constant / opcode breakdown + --compile recompile from .runar.ts instead of reading the golden + --compare a,b,c compile under several named variants and diff (implies --compile) + --json write machine-readable results + --top rows in the constant / opcode lists (default 8) + +Variants available: ${Object.keys(VARIANTS).join(', ')} +`.trim()); +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const fixtures = discoverFixtures(); + const selected = args.fixture + ? new Map(fixtures.has(args.fixture) ? [[args.fixture, fixtures.get(args.fixture)!]] : []) + : fixtures; + if (selected.size === 0) { + throw new Error(`no fixtures selected${args.fixture ? ` (unknown fixture '${args.fixture}')` : ''}`); + } + + if (args.compare.length > 0) { + for (const name of args.compare) { + if (!(name in VARIANTS)) { + throw new Error(`unknown variant '${name}'. Known: ${Object.keys(VARIANTS).join(', ')}`); + } + } + const byVariant = new Map>(); + const skipped: string[] = []; + for (const name of args.compare) { + const perFixture = new Map(); + for (const [fx] of selected) { + const src = tsSourcePath(fx); + if (src === null) { if (name === args.compare[0]) skipped.push(fx); continue; } + perFixture.set(fx, measureCompiled(fx, src, VARIANTS[name]!)); + } + byVariant.set(name, perFixture); + } + console.log(formatComparison(args.compare, byVariant)); + if (skipped.length > 0) { + // Never silently drop a fixture from a size report. + console.log(`\n_Skipped (no .runar.ts source): ${skipped.join(', ')}_`); + } + if (args.json) { + const payload = Object.fromEntries( + [...byVariant].map(([v, m]) => [v, Object.fromEntries(m)]), + ); + writeFileSync(args.json, `${JSON.stringify(payload, null, 2)}\n`); + } + return; + } + + const rows: FixtureMetrics[] = []; + const skipped: string[] = []; + for (const [fx, hexPath] of selected) { + if (args.compileMode) { + const src = tsSourcePath(fx); + if (src === null) { skipped.push(fx); continue; } + rows.push(measureCompiled(fx, src, VARIANTS.current!)); + } else { + rows.push(measureGolden(fx, hexPath)); + } + } + + if (args.detail) { + for (const r of [...rows].sort((a, b) => b.scriptBytes - a.scriptBytes)) { + console.log(formatDetail(r, args.top)); + console.log(''); + } + } else { + console.log(formatTable(rows)); + } + if (skipped.length > 0) { + console.log(`\n_Skipped (no .runar.ts source): ${skipped.join(', ')}_`); + } + if (args.json) { + writeFileSync(args.json, `${JSON.stringify(rows, null, 2)}\n`); + } +} + +const isMain = process.argv[1] !== undefined + && resolve(process.argv[1]).endsWith(join('runner', 'script-metrics.ts')); +if (isMain) { + try { + main(); + } catch (err) { + console.error(`script-metrics: ${(err as Error).message}`); + process.exit(1); + } +} + +export { REPO_ROOT }; diff --git a/docs/experiments/script-size-optimization-baseline.md b/docs/experiments/script-size-optimization-baseline.md new file mode 100644 index 000000000..e82e3956d --- /dev/null +++ b/docs/experiments/script-size-optimization-baseline.md @@ -0,0 +1,305 @@ +# Script-size optimization — Phase 0 baseline + +**Date:** 2026-08-28 · **Compiler:** TypeScript reference tier, default options (constant folding ON, EC optimizer ON, peephole ON) +**Source of truth:** the checked-in `conformance/tests//expected-script.hex` goldens, 72 fixtures, 13,526,545 bytes total. + +Reproduce every number here with: + +```bash +pnpm --filter runar-conformance run script-metrics # summary table +pnpm --filter runar-conformance run script-metrics -- --fixture p256-wallet --detail +pnpm --filter runar-conformance run script-metrics -- --json out.json # machine-readable +``` + +Instrumentation is read-only and does not change compilation output: +`packages/runar-compiler/src/metrics/script-metrics.ts` (byte classifier), +`packages/runar-compiler/src/metrics/cost-model.ts` (`estimateScriptBytes`, asserted +byte-exact against `06-emit.ts` over the whole corpus in `__tests__/cost-model.test.ts`), +`conformance/runner/script-metrics.ts` (the runner). + +--- + +## 1. The headline + +**58 % of every script byte Rúnar has ever emitted is a constant push, and 56 % of the +entire corpus is nine distinct constants pushed over and over.** + +| category | bytes | share of corpus | +|---|---:|---:| +| const-push | 7,840,690 | **58.0 %** | +| stack-shuffle | 3,133,223 | 23.2 % | +| arithmetic | 1,047,644 | 7.7 % | +| bytes (CAT/SPLIT/SIZE/EQUAL) | 878,631 | 6.5 % | +| small-int-push | 392,378 | 2.9 % | +| control | 160,000 | 1.2 % | +| crypto | 73,979 | 0.5 % | + +The nine largest single constants, all of them a curve's field prime `p` (or group order `n`): + +| fixture | constant | push size | pushes | bytes | share of that fixture | +|---|---|---:|---:|---:|---:| +| p384-wallet | P-384 `p` | 49 B | 30,577 | 1,528,850 | 77.9 % | +| p384-primitives | P-384 `p` | 49 B | 29,925 | 1,496,250 | 79.4 % | +| ec-primitives | secp256k1 `p` | 33 B | 28,102 | 955,468 | 71.7 % | +| ec-demo | secp256k1 `p` | 33 B | 28,102 | 955,468 | 71.7 % | +| **p256-wallet** | **P-256 `p`** | **33 B** | **20,025** | **680,850** | **71.0 %** | +| p256-primitives | P-256 `p` | 33 B | 19,755 | 671,670 | 72.4 % | +| schnorr-zkp | secp256k1 `p` | 33 B | 18,551 | 630,734 | 72.1 % | +| ec-unit | secp256k1 `p` | 33 B | 10,092 | 343,128 | 71.5 % | +| convergence-proof | secp256k1 `p` | 33 B | 9,547 | 324,598 | 71.8 % | + +Total: **7,587,016 bytes — 56 % of the corpus — spent re-pushing nine numbers.** + +### Why + +`cFieldMod` / `fieldMod` push the prime inline at *every* modular reduction: + +```ts +// packages/runar-compiler/src/passes/p256-p384-codegen.ts:135 +function cFieldMod(t: ECTracker, aName: string, resultName: string, c: CurveParams): void { + t.toTop(aName); + pushFieldP(t, '_fmod_p', c); // <-- 34 bytes (P-256) / 50 bytes (P-384), every time + t.rawBlock([aName, '_fmod_p'], resultName, (e) => { + e({ op: 'opcode', code: 'OP_2DUP' }); e({ op: 'opcode', code: 'OP_MOD' }); + e({ op: 'rot' }); e({ op: 'drop' }); e({ op: 'over' }); + e({ op: 'opcode', code: 'OP_ADD' }); e({ op: 'swap' }); e({ op: 'opcode', code: 'OP_MOD' }); + }); +} +``` + +Every `cFieldAdd` / `cFieldSub` / `cFieldMul` / `cFieldSqr` / `cFieldMulConst` ends in one of +these. `cEmitMul` unrolls 257 (P-256) / 385 (P-384) double-and-add rounds, `cFieldInv` and +`cGroupInv` unroll full Fermat ladders — so the prime push is multiplied by the unroll factor. + +A prime kept in a stack slot and copied with `push(depth); OP_PICK` costs **2–3 bytes** +instead of 34 or 50. Break-even is at two uses. + +--- + +## 2. p256-wallet — the brief's 959 kB reference, in detail + +`conformance/tests/p256-wallet` is **958,792 bytes**, which is the "959,592 B baseline +reference implementation" the optimization brief targets. It is a hybrid secp256k1 + P-256 +wallet: a P2PKH gate, then `verifyECDSA_P256(sig, p256Sig, p256PubKey)`. + +| category | bytes | share | +|---|---:|---:| +| const-push | 697,019 | 72.7 % | +| stack-shuffle | 173,967 | 18.1 % | +| arithmetic | 82,863 | 8.6 % | +| bytes | 1,210 | 0.1 % | +| small-int-push | 2,698 | 0.3 % | +| control | 1,031 | 0.1 % | +| crypto | 4 | 0.0 % | + +| repeated constant | size | pushes | bytes | share | +|---|---:|---:|---:|---:| +| P-256 field prime `p` | 33 B | 20,025 | 680,850 | 71.0 % | +| P-256 group order `n` | 33 B | 430 | 14,620 | 1.5 % | + +Top opcodes: `OP_MOD`×41,418 · `OP_ROT`×30,406 · `OP_SWAP`×27,342 · `OP_OVER`×23,417 · +`OP_DROP`×22,558 · `OP_ADD`×20,978 · `OP_2DUP`×20,453 · `OP_MUL`×13,214. + +Note the shape: **41,418 `OP_MOD` against 20,025 prime pushes** — two `OP_MOD` per reduction. +That is the sign-normalisation tail (`2DUP MOD ROT DROP OVER ADD SWAP MOD`), which exists +because `OP_MOD` takes the sign of the dividend. For a product of two values already reduced +into `[0, p)` the dividend is non-negative and the tail is dead weight: 6 of the 8 opcodes, +plus the second prime reference. That is a modular-domain-analysis win (brief Phase 4/5), not +a scheduling one. + +### Where the arithmetic actually goes + +Op-count goldens (`packages/runar-compiler/src/__tests__/p256-p384-codegen.test.ts:111`): + +| emitter | ops | measured bytes | +|---|---:|---:| +| `emitVerifyECDSA_P256` | 297,331 | 974,024 | +| `emitP256Mul` / `emitP256MulGen` | 140,036 / 140,038 | 459,746 / 459,812 | +| `emitP256Add` | 6,663 | 19,906 | +| `emitVerifyECDSA_P384` | 453,307 | 1,987,394 | +| `emitP384Mul` | 211,178 | 927,350 | + +`cEmitVerifyECDSA` runs **two independent 257-round ladders** (`u1·G` at `:1412`, `u2·Q` at +`:1442`) plus **three unrolled Fermat exponentiations** (`cFieldInv` 382 field muls, +`cGroupInv` 423, `cFieldPow` for the decompression sqrt 286). Nothing is shared between the +two ladders and no point is precomputed, even though `G` is a compile-time constant. + +--- + +## 3. Two populations, two different bottlenecks + +The corpus splits cleanly, and the split decides which optimization can touch which fixture. + +### EC / field-arithmetic fixtures — dominated by constants (72–80 % const-push) + +`p256-*`, `p384-*`, `ec-*`, `schnorr-zkp`, `convergence-proof`, `babybear*`. These scripts are +emitted by hand-written macro modules (`ec-codegen.ts`, `p256-p384-codegen.ts`, +`babybear-codegen.ts`, …) that build their own stack layout through `ECTracker` and its +clones. **They never pass through `05-stack-lower.ts`.** A generic ANF→Stack scheduler cannot +move a single byte of them. + +### Ordinary contracts — dominated by stack traffic (35–68 % stack-shuffle) + +Everything from `stateful-counter` (1,875 B) up through `math-demo` (17,348 B), and the small +fixtures most of all: `arithmetic` 67.9 %, `bounded-loop` 57.1 %, `multisig` 58.8 %, +`if-without-else-multi-temp` 55.3 %. These *are* produced by `05-stack-lower.ts`, and the +~30 % const-push in the mid-size stateful fixtures is largely BIP-143 sighash scaffolding, +not user data. + +### Hash / post-quantum fixtures — dominated by byte plumbing + +SLH-DSA (`OP_CAT`×80,120, `OP_SPLIT`×50,221 in the 128f fixture), SHA-256 and BLAKE3 sit at +2–17 % const-push, 36–44 % stack-shuffle, 23–28 % `bytes`. Constant pooling buys them almost +nothing; scheduling and byte-op fusion are the levers. + +| population | fixtures | const-push | stack-shuffle | reachable by | +|---|---|---:|---:|---| +| EC / field arithmetic | 9 (10.2 MB) | 72–80 % | 13–19 % | codegen-level constant pooling | +| hash / post-quantum | 12 (3.0 MB) | 2–17 % | 36–44 % | scheduling, byte-op fusion | +| ordinary contracts | 51 (0.1 MB) | 0–33 % | 35–68 % | generic liveness scheduler | + +--- + +## 4. Ranked byte sinks + +1. **Repeated field-prime pushes — 7,587,016 B (56 % of the corpus).** One pooled stack slot + per curve constant. Codegen-level (`ECTracker`), not a generic pass; the *policy* + (pool when `n_uses × push_cost > pool_cost + n_uses × pick_cost`) is generic and belongs + in the cost model. +2. **Stack traffic — 3,133,223 B (23 %).** Split roughly evenly between the EC macros' + `ECTracker.toTop`/`copyToTop` churn and `05-stack-lower.ts`'s `bringToTop`. The generic + half is addressable by liveness-driven scheduling; see + [`stack-scheduler-design.md`](stack-scheduler-design.md). +3. **The redundant second `OP_MOD` — ~20,000 reductions per EC fixture × 6 opcodes.** + Requires knowing an operand is already reduced (modular-domain analysis, brief Phase 4). +4. **Unrolled Fermat inversion.** 382 / 423 / 286 field muls per P-256 verify, three times. + An addition chain cuts each by ~30 %; a witness-supplied inverse (brief Phase 7) removes + them almost entirely. +5. **Two independent scalar ladders.** Straus/Shamir halves the doubling work; a fixed-base + comb for `u1·G` removes it (brief Phases 9–11). +6. **`emitReverse32` / `emitReverse48`.** 7 ops × 32 (or 48) per byte-order reversal, called + on every point decompose/compose. + +--- + +## 5. What Phase 0 already settles about the brief + +- **Phase 3 (fix-point peephole) is already done.** `optimizeStackIR` + (`optimizer/peephole.ts:507`) iterates `applyOnePass` to a fixed point with a 100-iteration + cap and recurses into `if` arms first. 28 rules, mirrored declaratively in + `optimizer/peephole-rules.ts` and executed pattern-vs-replacement through the `ScriptVM` by + `__tests__/peephole-exhaustive.test.ts`. What remains for Phase 3 is *more rules*, not a + fix-point driver. +- **Phase 15 (OP_PUSH_TX / CODESEPARATOR transaction binding) already ships** as + `passes/oppushtx-codegen.ts` — the BUG-100 fix derives the ECDSA signature from the pushed + preimage on-chain, so `OP_CHECKSIG` passes only when `hash256(preimage)` is the real + sighash. +- **Phase 2's generic scheduler cannot reach P-256.** See §3. The two must be prototyped + separately or the P-256 number will not move at all. +- **Phase 1's cost model is exact.** `estimateScriptBytes` agrees with `emitMethod` to the + byte on every method of all 67 fixtures that ship a `.runar.ts`, before and after peephole. + +--- + +## 6. Full corpus table + +Byte category shares per fixture, largest script first. + +| fixture | bytes | ops | const-push | stack-shuffle | arithmetic | bytes | crypto | small-int-push | control | other | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| p384-wallet | 1963300 | 430181 | 1565148 (79.7%) | 264286 (13.5%) | 126571 (6.4%) | 1754 (0.1%) | 4 (0.0%) | 3994 (0.2%) | 1543 (0.1%) | — | +| p384-primitives | 1883767 | 415719 | 1498750 (79.6%) | 256133 (13.6%) | 121250 (6.4%) | 1976 (0.1%) | — | 4109 (0.2%) | 1549 (0.1%) | — | +| ec-demo | 1332782 | 403869 | 957857 (71.9%) | 249884 (18.7%) | 113993 (8.6%) | 4208 (0.3%) | — | 5256 (0.4%) | 1584 (0.1%) | — | +| ec-primitives | 1332782 | 403869 | 957857 (71.9%) | 249884 (18.7%) | 113993 (8.6%) | 4208 (0.3%) | — | 5256 (0.4%) | 1584 (0.1%) | — | +| p256-wallet | 958792 | 282747 | 697019 (72.7%) | 173967 (18.1%) | 82863 (8.6%) | 1210 (0.1%) | 4 (0.0%) | 2698 (0.3%) | 1031 (0.1%) | — | +| p256-primitives | 928219 | 275241 | 673254 (72.5%) | 169769 (18.3%) | 80058 (8.6%) | 1336 (0.1%) | — | 2765 (0.3%) | 1037 (0.1%) | — | +| schnorr-zkp | 875189 | 262004 | 632257 (72.2%) | 162557 (18.6%) | 75250 (8.6%) | 1335 (0.2%) | 1 (0.0%) | 2759 (0.3%) | 1030 (0.1%) | — | +| post-quantum-slhdsa-192f | 788039 | 757325 | 54761 (6.9%) | 339659 (43.1%) | 56908 (7.2%) | 206316 (26.2%) | 17549 (2.2%) | 78196 (9.9%) | 34650 (4.4%) | — | +| post-quantum-slhdsa-256f | 729363 | 716143 | 19626 (2.7%) | 327166 (44.9%) | 57832 (7.9%) | 191936 (26.3%) | 17837 (2.4%) | 79752 (10.9%) | 35214 (4.8%) | — | +| post-quantum-slhdsa-128f | 533911 | 525388 | 12149 (2.3%) | 235416 (44.1%) | 39293 (7.4%) | 143273 (26.8%) | 12137 (2.3%) | 67751 (12.7%) | 23892 (4.5%) | — | +| ec-unit | 479716 | 146119 | 343994 (71.7%) | 89666 (18.7%) | 40878 (8.5%) | 2407 (0.5%) | — | 2248 (0.5%) | 523 (0.1%) | — | +| convergence-proof | 452386 | 136798 | 325416 (71.9%) | 84502 (18.7%) | 38705 (8.6%) | 1467 (0.3%) | — | 1780 (0.4%) | 516 (0.1%) | — | +| post-quantum-slhdsa-256s | 369173 | 358620 | 15558 (4.2%) | 162192 (43.9%) | 28402 (7.7%) | 96981 (26.3%) | 8818 (2.4%) | 40026 (10.8%) | 17196 (4.7%) | — | +| post-quantum-slhdsa-192s | 276583 | 262513 | 23837 (8.6%) | 116220 (42.0%) | 19195 (6.9%) | 72337 (26.2%) | 5985 (2.2%) | 27396 (9.9%) | 11613 (4.2%) | — | +| sphincs-wallet | 188609 | 183128 | 7747 (4.1%) | 80932 (42.9%) | 13282 (7.0%) | 50562 (26.8%) | 4164 (2.2%) | 23879 (12.7%) | 8043 (4.3%) | — | +| post-quantum-slhdsa | 188597 | 183116 | 7747 (4.1%) | 80926 (42.9%) | 13282 (7.0%) | 50560 (26.8%) | 4161 (2.2%) | 23878 (12.7%) | 8043 (4.3%) | — | +| sha256-finalize | 69507 | 63949 | 9898 (14.2%) | 24810 (35.7%) | 10155 (14.6%) | 16262 (23.4%) | — | 8379 (12.1%) | 3 (0.0%) | — | +| sha256-compress | 23145 | 21294 | 3296 (14.2%) | 8262 (35.7%) | 3384 (14.6%) | 5413 (23.4%) | — | 2790 (12.1%) | — | — | +| blake3 | 22428 | 20773 | 2798 (12.5%) | 8742 (39.0%) | 1811 (8.1%) | 6336 (28.3%) | — | 2738 (12.2%) | 3 (0.0%) | — | +| stateful-wots-gate | 20519 | 18177 | 3443 (16.8%) | 8025 (39.1%) | 2237 (10.9%) | 2454 (12.0%) | 1011 (4.9%) | 320 (1.6%) | 3029 (14.8%) | — | +| post-quantum-wallet | 19594 | 17514 | 3154 (16.1%) | 7690 (39.2%) | 2213 (11.3%) | 2275 (11.6%) | 1010 (5.2%) | 237 (1.2%) | 3015 (15.4%) | — | +| post-quantum-wots | 19582 | 17502 | 3154 (16.1%) | 7684 (39.2%) | 2213 (11.3%) | 2273 (11.6%) | 1007 (5.1%) | 236 (1.2%) | 3015 (15.4%) | — | +| math-demo | 17348 | 13183 | 4591 (26.5%) | 6188 (35.7%) | 1212 (7.0%) | 2842 (16.4%) | 62 (0.4%) | 1513 (8.7%) | 940 (5.4%) | — | +| babybear-ext4 | 5471 | 3084 | 2973 (54.3%) | 1320 (24.1%) | 1141 (20.9%) | — | — | 31 (0.6%) | 6 (0.1%) | — | +| function-patterns | 3844 | 2794 | 1159 (30.2%) | 1415 (36.8%) | 111 (2.9%) | 724 (18.8%) | 20 (0.5%) | 350 (9.1%) | 65 (1.7%) | — | +| token-ft | 3154 | 2330 | 929 (29.5%) | 1139 (36.1%) | 89 (2.8%) | 612 (19.4%) | 16 (0.5%) | 294 (9.3%) | 75 (2.4%) | — | +| merge-locals-shapes | 3031 | 2236 | 882 (29.1%) | 1145 (37.8%) | 88 (2.9%) | 567 (18.7%) | 12 (0.4%) | 280 (9.2%) | 57 (1.9%) | — | +| private-helper-outputs | 2879 | 2080 | 886 (30.8%) | 1018 (35.4%) | 76 (2.6%) | 559 (19.4%) | 12 (0.4%) | 271 (9.4%) | 57 (2.0%) | — | +| assert-false-guard | 2033 | 1507 | 582 (28.6%) | 778 (38.3%) | 54 (2.7%) | 378 (18.6%) | 8 (0.4%) | 186 (9.1%) | 47 (2.3%) | — | +| loop-if-merged-locals | 2011 | 1481 | 588 (29.2%) | 750 (37.3%) | 60 (3.0%) | 378 (18.8%) | 8 (0.4%) | 186 (9.2%) | 41 (2.0%) | — | +| terminal-varlen-read | 1940 | 1415 | 584 (30.1%) | 688 (35.5%) | 55 (2.8%) | 375 (19.3%) | 7 (0.4%) | 178 (9.2%) | 53 (2.7%) | — | +| property-initializers | 1878 | 1354 | 578 (30.8%) | 674 (35.9%) | 50 (2.7%) | 362 (19.3%) | 8 (0.4%) | 175 (9.3%) | 31 (1.7%) | — | +| stateful | 1876 | 1352 | 578 (30.8%) | 674 (35.9%) | 50 (2.7%) | 362 (19.3%) | 8 (0.4%) | 174 (9.3%) | 30 (1.6%) | — | +| stateful-counter | 1875 | 1351 | 578 (30.8%) | 673 (35.9%) | 51 (2.7%) | 362 (19.3%) | 8 (0.4%) | 173 (9.2%) | 30 (1.6%) | — | +| stateful-bytestring | 1851 | 1335 | 569 (30.7%) | 660 (35.7%) | 48 (2.6%) | 357 (19.3%) | 8 (0.4%) | 171 (9.2%) | 38 (2.1%) | — | +| auction | 1794 | 1288 | 553 (30.8%) | 656 (36.6%) | 47 (2.6%) | 346 (19.3%) | 9 (0.5%) | 164 (9.1%) | 19 (1.1%) | — | +| token-nft | 1738 | 1234 | 549 (31.6%) | 630 (36.2%) | 43 (2.5%) | 332 (19.1%) | 9 (0.5%) | 157 (9.0%) | 18 (1.0%) | — | +| state-covenant | 1196 | 912 | 326 (27.3%) | 451 (37.7%) | 41 (3.4%) | 221 (18.5%) | 9 (0.8%) | 105 (8.8%) | 43 (3.6%) | — | +| branched-readonly-len | 1096 | 816 | 319 (29.1%) | 392 (35.8%) | 33 (3.0%) | 211 (19.3%) | 4 (0.4%) | 100 (9.1%) | 37 (3.4%) | — | +| conditional-data-output-stateful | 1015 | 740 | 308 (30.3%) | 356 (35.1%) | 27 (2.7%) | 197 (19.4%) | 4 (0.4%) | 97 (9.6%) | 26 (2.6%) | — | +| merge-locals-prop-updates | 1006 | 741 | 294 (29.2%) | 387 (38.5%) | 27 (2.7%) | 189 (18.8%) | 4 (0.4%) | 89 (8.8%) | 16 (1.6%) | — | +| add-raw-output | 1005 | 728 | 311 (30.9%) | 349 (34.7%) | 27 (2.7%) | 197 (19.6%) | 4 (0.4%) | 95 (9.5%) | 22 (2.2%) | — | +| add-data-output | 1004 | 729 | 308 (30.7%) | 351 (35.0%) | 27 (2.7%) | 197 (19.6%) | 4 (0.4%) | 95 (9.5%) | 22 (2.2%) | — | +| selector | 985 | 723 | 289 (29.3%) | 371 (37.7%) | 28 (2.8%) | 185 (18.8%) | 4 (0.4%) | 88 (8.9%) | 20 (2.0%) | — | +| branch-merged-locals | 963 | 699 | 292 (30.3%) | 354 (36.8%) | 24 (2.5%) | 185 (19.2%) | 4 (0.4%) | 88 (9.1%) | 16 (1.7%) | — | +| cond-write-multi-field | 957 | 693 | 292 (30.5%) | 345 (36.1%) | 26 (2.7%) | 185 (19.3%) | 4 (0.4%) | 89 (9.3%) | 16 (1.7%) | — | +| state-bigint-edges | 952 | 688 | 292 (30.7%) | 346 (36.3%) | 25 (2.6%) | 185 (19.4%) | 4 (0.4%) | 87 (9.1%) | 13 (1.4%) | — | +| intent-current-block-height | 944 | 682 | 289 (30.6%) | 338 (35.8%) | 26 (2.8%) | 185 (19.6%) | 4 (0.4%) | 88 (9.3%) | 14 (1.5%) | — | +| intent-prev-output-script | 942 | 680 | 289 (30.7%) | 339 (36.0%) | 25 (2.7%) | 183 (19.4%) | 5 (0.5%) | 87 (9.2%) | 14 (1.5%) | — | +| oversize-bigint-shift | 940 | 671 | 297 (31.6%) | 334 (35.5%) | 25 (2.7%) | 181 (19.3%) | 4 (0.4%) | 86 (9.1%) | 13 (1.4%) | — | +| state-ripemd160 | 931 | 668 | 291 (31.3%) | 336 (36.1%) | 23 (2.5%) | 179 (19.2%) | 4 (0.4%) | 84 (9.0%) | 14 (1.5%) | — | +| intent-output-p2pkh | 843 | 594 | 270 (32.0%) | 309 (36.7%) | 18 (2.1%) | 165 (19.6%) | 4 (0.5%) | 76 (9.0%) | 1 (0.1%) | — | +| covenant-vault | 795 | 550 | 262 (33.0%) | 290 (36.5%) | 13 (1.6%) | 151 (19.0%) | 5 (0.6%) | 73 (9.2%) | 1 (0.1%) | — | +| all-readonly-cleanstack | 777 | 539 | 252 (32.4%) | 288 (37.1%) | 14 (1.8%) | 146 (18.8%) | 4 (0.5%) | 72 (9.3%) | 1 (0.1%) | — | +| babybear | 647 | 351 | 370 (57.2%) | 99 (15.3%) | 156 (24.1%) | — | — | 7 (1.1%) | 15 (2.3%) | — | +| if-without-else-multi-temp | 226 | 219 | 11 (4.9%) | 125 (55.3%) | 15 (6.6%) | 24 (10.6%) | — | 25 (11.1%) | 26 (11.5%) | — | +| merkle-proof | 201 | 193 | 16 (8.0%) | 108 (53.7%) | 16 (8.0%) | 18 (9.0%) | 8 (4.0%) | 16 (8.0%) | 19 (9.5%) | — | +| bitwise-ops | 96 | 96 | — | 34 (35.4%) | 26 (27.1%) | — | — | 27 (28.1%) | 9 (9.4%) | — | +| cross-covenant | 46 | 45 | 2 (4.3%) | 24 (52.2%) | 3 (6.5%) | 8 (17.4%) | 2 (4.3%) | 4 (8.7%) | 3 (6.5%) | — | +| oracle-price | 44 | 38 | 8 (18.2%) | 21 (47.7%) | 5 (11.4%) | 2 (4.5%) | 2 (4.5%) | 4 (9.1%) | 2 (4.5%) | — | +| bounded-loop | 42 | 42 | — | 24 (57.1%) | 11 (26.2%) | — | — | 7 (16.7%) | — | — | +| arithmetic | 28 | 28 | — | 19 (67.9%) | 8 (28.6%) | — | — | 1 (3.6%) | — | — | +| if-without-else | 27 | 27 | — | 10 (37.0%) | 5 (18.5%) | — | — | 6 (22.2%) | 6 (22.2%) | — | +| shift-ops | 27 | 27 | — | 10 (37.0%) | 8 (29.6%) | — | — | 8 (29.6%) | 1 (3.7%) | — | +| if-else | 20 | 20 | — | 10 (50.0%) | 3 (15.0%) | — | — | 4 (20.0%) | 3 (15.0%) | — | +| escrow | 19 | 19 | — | 4 (21.1%) | 2 (10.5%) | — | 4 (21.1%) | 6 (31.6%) | 3 (15.8%) | — | +| multi-method | 19 | 19 | — | 2 (10.5%) | 5 (26.3%) | — | 2 (10.5%) | 6 (31.6%) | 4 (21.1%) | — | +| multisig | 17 | 17 | — | 10 (58.8%) | — | — | 1 (5.9%) | 6 (35.3%) | — | — | +| boolean-logic | 15 | 15 | — | 6 (40.0%) | 7 (46.7%) | — | — | 2 (13.3%) | — | — | +| go-dsl-bytestring-literal | 8 | 6 | 3 (37.5%) | — | 2 (25.0%) | 1 (12.5%) | — | 2 (25.0%) | — | — | +| basic-p2pkh | 5 | 5 | — | 1 (20.0%) | — | 1 (20.0%) | 2 (40.0%) | 1 (20.0%) | — | — | +| asm-raw-script | 1 | 1 | — | — | — | — | — | 1 (100.0%) | — | — | +_`asm-raw-script` is a single opaque `raw_bytes` span; `basic-p2pkh` is the 5-byte template +before constructor-arg splicing. Neither is a size target._ + +--- + +## 7. Method + +`analyzeScriptHex` walks the serialized script and charges every byte to exactly one +category; the sum is asserted equal to the script length. One rule is worth stating: a push +immediately consumed by `OP_PICK` / `OP_ROLL` is charged to **stack-shuffle**, not to +**const-push**. `bringToTop` emits `push(depth)` + `OP_PICK` as a pair +(`05-stack-lower.ts:1062`), and charging those depth bytes to constants would credit the +wrong optimizer with removing them. On `p256-wallet` that reclassification moves exactly +21,926 bytes, and it carries a second useful fact: all 21,926 of those depth pushes are a +single byte, so the EC macros never `OP_PICK` deeper than 16. A pooled constant parked below +a working set that shallow would cost 3 bytes to copy (2-byte depth push + `OP_PICK`) instead +of 34 — still a 31-byte saving per reduction. + +Categories: `const-push` (length-prefixed / PUSHDATA payloads), `small-int-push` +(OP_0/OP_1NEGATE/OP_1..16), `stack-shuffle` (DUP/DROP/NIP/OVER/PICK/ROLL/ROT/SWAP/TUCK/2DROP/ +2DUP/3DUP/2OVER/2ROT/2SWAP/IFDUP/DEPTH/TOALTSTACK/FROMALTSTACK plus PICK/ROLL depth pushes), +`arithmetic` (numeric, bitwise and comparison opcodes), `bytes` (CAT/SPLIT/SIZE/NUM2BIN/ +BIN2NUM/SUBSTR/LEFT/RIGHT/EQUAL/EQUALVERIFY), `crypto` (hashes, CHECKSIG family, +CODESEPARATOR), `control` (IF/NOTIF/ELSE/ENDIF/VERIFY/RETURN/NOP/CLTV/CSV), `other`. diff --git a/docs/experiments/script-size-optimizer-results.md b/docs/experiments/script-size-optimizer-results.md new file mode 100644 index 000000000..ca780d448 --- /dev/null +++ b/docs/experiments/script-size-optimizer-results.md @@ -0,0 +1,365 @@ +# Script-size optimizer — Phases 0–2 results + +**Scope:** the first slice of the size-optimization brief — baseline instrumentation (Phase 0), +an exact script-byte cost model (Phase 1), and two prototype optimizations behind opt-in flags +(Phase 2). No modular-domain analysis, no witness hints, no scalar-multiplication algorithms. +**Companion documents:** [`script-size-optimization-baseline.md`](script-size-optimization-baseline.md), +[`stack-scheduler-design.md`](stack-scheduler-design.md). + +Everything below is measured on the 72 conformance fixtures with +`pnpm --filter runar-conformance run script-metrics -- --compare current,liveness,ec-pool,both`. + +--- + +## 1. Headline + +| | corpus bytes | vs baseline | fixtures changed | fixtures grown | +|---|---:|---:|---:|---:| +| `current` (shipping) | 13,526,563 | — | — | — | +| `liveness` (scheduler) | 13,526,482 | −0.0 % | 34 | 0 | +| `ec-pool` (constant pool) | 6,285,154 | **−53.5 %** | 9 | 0 | +| `both` | 6,285,073 | **−53.5 %** | 43 | 0 | + +**`conformance/tests/p256-wallet`: 958,792 → 304,463 bytes (−68.2 %)**, which is the brief's +959 kB reference implementation. `p384-wallet`: 1,963,300 → 463,435 (−76.4 %). + +The next step after this slice has been measured rather than projected: adding reduction +sinking takes `p256-wallet` to **179,796 bytes (−81.2 % from shipping)** and `p384-wallet` to +272,584 (−86.1 %). See §3.7, and §3.8 for the precondition it turns out to need. + +Default output is unchanged: all 72 fixtures still reproduce their checked-in +`expected-script.hex` byte-for-byte +(`packages/runar-compiler/src/__tests__/golden-invariance.test.ts`), and the Go and Rust +cross-compiler golden tests still pass. + +--- + +## 2. What was built + +| Phase | Deliverable | Files | +|---|---|---| +| 0 | Byte-category instrumentation + benchmark runner | `packages/runar-compiler/src/metrics/script-metrics.ts`, `conformance/runner/script-metrics.ts` | +| 1 | Exact script-byte cost model | `packages/runar-compiler/src/metrics/cost-model.ts` | +| 2a | Liveness scheduler (`--stack-scheduler=liveness`) | `packages/runar-compiler/src/passes/05-stack-lower.ts` | +| 2b | EC constant pool (`--ec-constant-pool`) | `packages/runar-compiler/src/passes/ec-codegen.ts`, `p256-p384-codegen.ts` | + +### Phase 1 — the cost model is exact, not an estimate + +```ts +estimateScriptBytes(ops) === emitMethod({ ops, … }).scriptHex.length / 2 +``` + +is asserted for every method of every fixture that ships a `.runar.ts`, before and after +peephole (`__tests__/cost-model.test.ts`, 101 assertions). That exactness turned out to matter +more than expected — see §5. + +### Phase 2b — EC constant pool + +`ECTracker` gained a pooled slot per curve constant. `pushConst(slot, value, name)` compares +the emitted cost of `OP_PICK`-ing the slot against re-pushing the literal and takes the +cheaper, so pooling can never make an individual call site larger. Every EC emitter that does +more than a couple of reductions parks the field prime (and, for the ladders, the group order) +on entry and releases it on exit. + +| emitter | current | pooled | | +|---|---:|---:|---:| +| `emitVerifyECDSA_P256` | 974,024 | 319,693 | −67.2 % | +| `emitVerifyECDSA_P384` | 1,987,394 | 487,527 | −75.5 % | +| `emitP256Mul` | 459,746 | 150,512 | −67.3 % | +| `emitP384Mul` | 927,350 | 227,044 | −75.5 % | +| `emitEcMul` (secp256k1) | 428,676 | 140,242 | −67.3 % | +| `emitEcAdd` | 25,426 | 8,791 | −65.4 % | + +### Phase 2a — liveness scheduler + +Two transformations, both gated on `schedulerMode: 'liveness'`: + +1. **Alt-stack result spilling.** A result whose next use is not the following binding is + parked with `OP_TOALTSTACK`, keeping the operands a chain reads repeatedly at depth 0/1. + The whole spill group is restored in one go before the first binding that needs any of it — + which puts the values back in production order, the order an ANF accumulation reads them. +2. **Commutative operand ordering.** For `+ * === !== & | ^` (excluding `+` on ByteString, + which is `OP_CAT`), the operands are materialized in whichever order the cost model scores + cheaper. + +Selection is per method: both schedules are lowered and the cheaper one — measured after +peephole — is kept. "The scheduler never grows a method" is therefore structural, not a hope. + +| fixture | current | liveness | | +|---|---:|---:|---:| +| `arithmetic` | 28 | 18 | **−35.7 %** | +| `bounded-loop` | 42 | 37 | −11.9 % | +| `if-without-else-multi-temp` | 244 | 243 | −0.4 % | +| ~30 mid-size fixtures | 795–17,348 | | −0.1 % to −0.2 % | +| EC / SLH-DSA / SHA-256 / BLAKE3 | | unchanged | 0 % | + +18 bytes for `arithmetic` is the hand-derived optimum recorded in +[`stack-scheduler-design.md`](stack-scheduler-design.md) §2.4 — the scheduler reached it +independently. + +--- + +## 3. Answers to the brief's questions (for this slice) + +### 3.1 What is generic? + +- **The byte-cost model.** Nothing else in the compiler could compare two lowerings by the + metric that matters. It should become permanent infrastructure regardless of which + optimizations ship. +- **Cost-model-driven selection.** Lowering a method both ways and keeping the smaller is + cheap, obviously correct, and removes a whole class of "the heuristic guessed wrong" + regressions. Recommended as the general pattern for any future scheduling change. +- **Commutative operand ordering.** Small, local, and applies to every contract. +- **Constant pooling as a *policy*** — "if a value is materialized more than + `pool_cost / (push_cost − pick_cost)` times, park it" — is fully generic even though this + implementation applies it inside the EC macros. + +### 3.2 What needs explicit programmer intent? + +Nothing in this slice. Both flags are compiler-internal and semantics-preserving. Witness +hints (brief Phase 7) remain the first thing that genuinely requires a contract-level opt-in. + +### 3.3 What belongs in crypto libraries / intrinsics? + +The pooling *mechanism* had to live in `ECTracker`, because the crypto emitters build their +own stack layout and never pass through `05-stack-lower.ts`. That boundary is the single most +important architectural fact this slice established — see §4. + +### 3.4 What is P-256-specific? + +Nothing that was built. `POOL_FIELD_P` / `POOL_GROUP_N` are parameterized by `CurveParams` / +`GroupParams`; the same code path serves secp256k1, P-256 and P-384, and the −67 % / −76 % +results differ only because P-384's prime is a 50-byte push instead of 34. + +### 3.5 What should not be implemented? + +- **Eager dead-slot retirement.** The plan assumed dead values sink and inflate later + `OP_PICK` depth pushes. Measurement killed it: of 387,749 `OP_PICK`/`OP_ROLL` sites in the + corpus, **657 are deeper than 16**; typical depths are 2–5, and depths 0–2 are single-byte + opcodes anyway. Every drop would cost 1–3 bytes to save approximately nothing. +- **A forked scheduler pass.** A `schedulerMode` field on `LoweringContext` kept one code + path and let the existing 66 structural assertions in `05-stack-lower.test.ts` cover both + modes. A 5,500-line fork would have drifted from the branch/loop invariant fixes landing on + the original. + +### 3.6 Can Rúnar approach the ~29.6 kB P-256 result? + +Not yet — `p256-wallet` is at 304,463 bytes, 10× the target. But the remaining bytes are in +exactly two buckets, neither is mysterious, and the size of the next step has been **measured** +rather than projected (see §3.7). + +| category | bytes | share | +|---|---:|---:| +| stack-shuffle | 214,904 | 70.6 % | +| arithmetic | 82,863 | 27.2 % | +| small-int push | 2,698 | 0.9 % | +| const-push | 1,753 | 0.6 % | +| everything else | 2,245 | 0.7 % | + +Constant pushes went from 697,019 bytes to 1,753 — 99.7 % eliminated. What is left is the +modular-reduction sequence itself, repeated ~20,000 times: + +``` +pick p 2 bytes (was a 34-byte push) +OP_2DUP OP_MOD OP_ROT OP_DROP OP_OVER OP_ADD OP_SWAP OP_MOD 8 bytes +``` + +The six-opcode tail after the first `OP_MOD` exists only because `OP_MOD` takes the sign of +the dividend. Where the dividend is provably non-negative the tail is dead weight. + +### 3.7 Measured ceiling for reduction sinking + +Rather than project the next step, it was measured directly: `fieldMod` / `cFieldMod` were +patched behind a throwaway env switch to emit the short form, the corpus was re-measured, and +the patch was discarded. Two variants: + +- `nonneg` — short reduction where the dividend is provably ≥ 0 (`fieldMul`, `fieldSqr`, + `fieldAdd`, `fieldMulConst`), and for `fieldSub` the cheap `a − b + p` then one `OP_MOD` + (6 bytes instead of 10). This is what a correct analysis could actually emit. +- `all` — short reduction everywhere. Semantically wrong; the absolute floor. + +| fixture | shipping | + pool | **+ pool + sinking** | floor (`all`) | +|---|---:|---:|---:|---:| +| `p256-wallet` | 958,792 | 304,463 | **179,796** (−81.2 %) | 164,302 | +| `p384-wallet` | 1,963,300 | 463,435 | **272,584** (−86.1 %) | 249,410 | +| `ec-primitives` | 1,332,782 | 433,880 | **258,160** (−80.6 %) | 237,229 | +| `ec-unit` | 479,716 | 157,129 | **93,585** (−80.5 %) | 86,576 | + +**The sound variant captures 89 % of the theoretical floor** (124,667 of a possible 140,161 +bytes on `p256-wallet`), so the analysis does not need to be clever about subtraction — the +cheap `+p` form is nearly free. + +Note the pooling and the sinking are complementary, not independent: without pooling, the +cheap `fieldSub` form pushes the prime *twice* and `p256-wallet` gets **larger** (958,792 → +999,371). Sinking only pays once the prime is a 2-byte pick. + +### 3.8 The analysis needed is a sign lattice, not a modular-domain lattice + +The `nonneg` variant passes **256 EC oracle assertions** — OpenSSL signatures on both curves, +`ec-on-curve-canonicity`, `ec-degenerate-add`, `ec-mul-scalars`, `p256-p384-scalars`, +`p256-p384-ecdsa-verify`. It would have shipped looking green. + +It is nonetheless unsound, in a narrow and precisely characterised window: + +- The **multiply / add / mulconst** paths need only *dividend ≥ 0*. That is already implied by + `OP_BIN2NUM` of unsigned coordinate bytes, by products of non-negatives, and by sums of + non-negatives. Roughly 70 % of all reductions qualify under a trivial sign analysis. +- The **subtract** path needs the strictly stronger *subtrahend < p*, and that is NOT implied + by "decoded from 32 unsigned bytes". + +The concrete divergence, found by construction: + +``` +ecAdd((0, 1), (2^256 − 1, 1)) + shipping : fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85f… + sinking : 00000000000000000000000000000000000000000000000001000003d0… + ^ 0x1000003d0 = 2^32 + 977 = 2^256 − p +``` + +It bites only when the subtrahend is non-canonical *and* the minuend is smaller than the gap +between `p` and `2^256` — a ~2^32-wide window out of 2^256, reachable only through the +unguarded bare builtins (`ecAdd`, `p256Add`, `p256Mul` take raw coordinates; +`verifyECDSA_*` and `onCurve` run a canonicity guard first). + +So the requirement for Phase 4/5 is sharper than "modular-domain analysis": a **sign lattice** +plus a **`< p` bit that only subtrahends need**. That is a materially smaller piece of work +than a full domain lattice, and it is the difference between an optimization that passes 256 +oracle assertions and one that is actually correct. + +### 3.9 Revised trajectory + +``` +958,792 shipping +179,796 + constant pool + reduction sinking (MEASURED) + − Straus/Shamir: one joint ladder instead of two (Phase 9, estimated) + − fixed-base comb for u1·G, G compile-time known (Phases 10–11, estimated) + ~30,000 ← the reference trajectory's comb stage (34,470 B) +``` + +Everything above 179,796 is measured; everything below it is still an estimate. + +Note what the measurement does to the ordering. At 179,796 bytes the split is 69.6 % +stack-shuffle / 26.7 % arithmetic, and `OP_PICK` (×36,683) is the single largest opcode. Once +a reduction costs 3 bytes, **`ECTracker`'s own operand shuffling is the bottleneck, not the +reduction**. Straus and comb still help — they cut total operations, so both columns shrink — +but the structural fix underneath is the field-element IR in §4: it would put those 125,119 +bytes of shuffle within reach of a scheduler, which nothing can reach today. + +--- + +## 4. The architectural finding + +**Crypto codegen does not pass through the generic backend.** `ec-codegen.ts`, +`p256-p384-codegen.ts`, `sha256-codegen.ts`, `slh-dsa-codegen.ts`, `babybear-codegen.ts` and +their peers emit Stack IR directly through hand-written `ECTracker`-family trackers. +`05-stack-lower.ts` never sees those ops. + +Consequences, all confirmed by measurement: + +- A generic scheduler cannot move a single byte of the EC, SLH-DSA, SHA-256 or BLAKE3 + fixtures — 13.4 MB of the 13.5 MB corpus. +- Conversely, the constant pool cannot help ordinary contracts. +- Any future "generic" optimization needs an answer to which of the two worlds it lives in + before its value can be estimated. + +The peephole optimizer is the one pass that spans both, because it runs on the whole method's +Stack IR after lowering. That makes new peephole rules unusually high-leverage — and it is +already a fix-point driver (brief Phase 3 is done; see the baseline document §5). + +--- + +## 5. What the oracles caught + +The scheduler's first working version **miscompiled `if-without-else-multi-temp`**: it +produced a script that ran to completion, left a truthy top-of-stack, and **accepted a witness +the shipping compiler rejects**. Byte counts, golden comparisons for the other fixtures, and +the compiler's own 4,099 unit tests all passed while this was true. + +What caught it was `conformance/witnesses/` replayed through `runDifferentialExecution` — +source-semantics interpreter versus deployed script, on witnesses the repo had already +committed to, with at least one accept and one reject per fixture. Two of 86 cases failed. + +Root cause: restoring spilled values immediately before an `if` leaves the parent stack in a +shape `lowerIf`'s arm reconciliation was not written for. The fix is a precondition — spilling +is refused in any scope that still has control flow ahead of it — not an attempt to make the +two agree. That costs the scheduler its wins on `oracle-price` and `cross-covenant` (2 and 1 +bytes) and keeps everything else. + +Two process notes worth carrying forward: + +- A bisect that "passes" can be vacuous. Disabling commutative reordering made the failure + disappear, which looked like an acquittal for spilling — but with reordering off, the + method-level cost guard simply preferred the baseline schedule and *no spilling happened at + all*. Confirming that the variant actually changed the bytes was what made the second + bisect meaningful. +- The per-site cost model was wrong until it was made peephole-aware. Two consumed operands at + depths 1 and 0 emit `OP_SWAP OP_SWAP`, which the `swap-swap` rule deletes outright — free — + while the "cheaper-looking" order emits one real `OP_SWAP`. Scoring candidate op sequences + through `optimizeStackIR` before comparing them took `arithmetic` from 24 bytes to 18. + +--- + +## 6. Recommended compiler changes + +**Adopt now (independent of any optimization):** + +1. `packages/runar-compiler/src/metrics/cost-model.ts` and `script-metrics.ts` as permanent + infrastructure, with the exactness sweep as a standing test. +2. `conformance/runner/script-metrics.ts` alongside the existing `script-size-check.ts` — the + latter answers "did anything grow?", the former "where did the bytes go?". + +**Adopt after a 7-tier port:** + +3. The EC constant pool. It is the largest single byte win available, it is + curve-parameterized rather than curve-specific, and it is proved equivalent against OpenSSL + signatures on both curves plus every SEC1 rejection case + (`packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts`, 44 cases). + Landing it means porting to `compilers/{go,rust,python,ruby,zig,java}`, regenerating 9 + goldens, re-stamping `conformance/script-size-baseline.json` (the −67 % shrink trips its + 50 % guard by design), and adding provenance entries. + +**Keep experimental:** + +4. The liveness scheduler. It is correct and never grows a fixture, but outside small + arithmetic contracts it buys 0.1 %. Not worth a 7-tier port on its own; worth keeping as a + gated mode so the next scheduling idea has somewhere to land. + +**Do next (highest value first):** + +5. **Sign analysis + reduction sinking** (brief Phases 4–5). **Measured at 124,667 bytes on + `p256-wallet`** (304,463 → 179,796) — see §3.7. Two facts to carry: *dividend ≥ 0*, which a + trivial sign lattice gives for ~70 % of reductions, and *subtrahend < p*, which only + `fieldSub` needs and which unsigned 32-byte decoding does NOT imply (§3.8). It is also the + prerequisite for everything after it: Straus, comb and lazy accumulation all need to know + which values are already reduced. +6. **A typed field-element IR under the crypto emitters.** After §5 lands, 69.6 % of what + remains is `ECTracker`'s own operand shuffling, which no pass can currently reach. This is + larger than any single item here, and it would let §5, §7 and future work be written once + instead of seven times (§4). +7. **Straus/Shamir joint ladder**, then a **fixed-base comb** for `u1·G` (Phases 9–11). +8. **Witness-hint modular inverse** (Phase 7) — removes three unrolled Fermat ladders + (382 + 423 + 286 field multiplications per P-256 verify). The first item requiring an + explicit soundness argument rather than a translation-validation proof. + +--- + +## 7. Reproducing + +```bash +# Sizes +pnpm --filter runar-conformance run script-metrics # baseline table +pnpm --filter runar-conformance run script-metrics -- --fixture p256-wallet --detail +pnpm --filter runar-conformance run script-metrics -- --compare current,liveness,ec-pool,both + +# Correctness +npx vitest run packages/runar-compiler/src/__tests__/cost-model.test.ts # model is exact +npx vitest run packages/runar-compiler/src/__tests__/golden-invariance.test.ts # default unchanged +npx vitest run packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts +npx vitest run packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts +npx vitest run packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts +npx vitest run packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts +npx vitest run packages/runar-testing/src/__tests__/scheduler-headroom.test.ts + +# CLI +node --import tsx packages/runar-cli/src/bin.ts compile --ec-constant-pool --hex +``` diff --git a/docs/experiments/stack-scheduler-design.md b/docs/experiments/stack-scheduler-design.md new file mode 100644 index 000000000..4905e4b97 --- /dev/null +++ b/docs/experiments/stack-scheduler-design.md @@ -0,0 +1,393 @@ +# Stack scheduler — current behaviour, measured inefficiencies, and a design + +**Status:** design, pre-implementation. Companion to +[`script-size-optimization-baseline.md`](script-size-optimization-baseline.md). +**Scope:** the generic ANF → Stack lowering in +`packages/runar-compiler/src/passes/05-stack-lower.ts`. Crypto macro modules +(`ec-codegen.ts`, `p256-p384-codegen.ts`, `sha256-codegen.ts`, `slh-dsa-codegen.ts`, …) +emit Stack IR directly through their own `ECTracker`-family trackers and are **out of +scope for this document** — they are addressed separately by constant pooling. + +--- + +## 1. What the current lowering does + +### 1.1 The symbolic stack + +`class StackMap` (`05-stack-lower.ts:166`) is an array of `(string | null)`, index 0 = +bottom. `null` marks an anonymous slot (e.g. the discard half of an `OP_SPLIT`). + +```ts +findDepth(name): number // :192 — searches TOP-DOWN, returns depth-from-top +removeAtDepth(d) // :207 — the ROLL effect +peekAtDepth(d) // :217 — the PICK effect +clone() // :226 — used to fork a branch arm +``` + +`findDepth` resolves to the *shallowest* match, so rebinding a name pushes a new slot with +the same name and the old one becomes dead-but-resident. `LoweringContext` (`:547`) owns one +`StackMap`, the emitted `ops`, `maxDepth`, and `outerProtectedRefs`. The constructor (`:578`) +seeds the map with parameter names, first param at the bottom. + +### 1.2 Liveness + +The entire liveness analysis is `computeLastUses(bindings)` (`:276`): one forward scan +mapping each referenced name → the highest binding index that references it. Array-literal +indirection is patched through (`:284`) so element temps stay live to the array's consumer. + +Consumption is decided by two predicates: + +```ts +isLastUse(ref, i, lastUses) // :1304 last <= i +operandConsume(ref, operands, i, …) // :1328 isLastUse AND appears once in this operand list +``` + +`operandConsume` needs the occurrence check because `t := x + x` must PICK at both positions. + +Outer-scope values are pinned by *forcing* their last use past the end: +`lastUses.set(ref, bindings.length)` (`:1154`, and `:2196` for branch arms). That is the only +pinning mechanism. + +### 1.3 Materialization — `bringToTop(name, consume)` (`:1038`) + +Every operand goes through this one function: + +| depth | consume (last use) | !consume (still live) | +|---:|---|---| +| 0 | nothing | `OP_DUP` | +| 1 | `OP_SWAP` | `OP_OVER` | +| 2 | `OP_ROT` | `push 2; OP_PICK` | +| d | `push d; OP_ROLL` | `push d; OP_PICK` | + +The depth 0/1/2 peepholes are inlined here, which is why the peephole rules `roll1-to-swap`, +`roll2-to-rot`, `pick0-to-dup`, `pick1-to-over` almost never fire on the main path. + +### 1.4 What is *not* done + +- **No operand reordering.** `lowerBinOp` (`:1507`) always materializes left then right, + even for commutative operators, and `lowerCall` (`:1839`) always walks args in order. +- **No proactive dead-value removal.** `computeLastUses` knows exactly when a temp dies; + nothing acts on it. Dead slots linger until `cleanupExcessStack()` (`:621`) NIPs the method + tail, or until a branch's `drainBranchPrivateResidue` (`:1112`) sweeps them. +- **No alt stack.** The generic lowerer emits `OP_TOALTSTACK` in exactly three places + (`:3013`, `:3105` state serialization, and the `divmod` intrinsic at `:4517`). Never for + scheduling. `docs/compiler-architecture.md` claims otherwise — that paragraph is + aspirational and should be corrected. +- **No cost model.** Choices are structural, never compared by emitted bytes. + +### 1.5 Branches and loops, in one line each + +`lowerIf` (`:2092`) forks a cloned `StackMap` per arm, pins every parent value that outlives +the `if`, reconciles asymmetric consumption, trims to the declared `results` layout, pads the +shallower arm with 1-byte empty pushes (`:2400`, `:2405`), and asserts equal arm depth at +`OP_ENDIF`. `lowerLoop` (`:2665`) fully unrolls, recomputing `lastUses` per iteration and +pinning loop-carried refs on every non-final iteration. + +Any scheduler change must leave these invariants intact — they are enforced by hard throws +(`branch result layout mismatch` at `:2346`, the Layer B/C depth assertions at `:2417`, +`:2640`), not by tests alone. + +--- + +## 2. Measured inefficiencies + +All figures from the 72 checked-in goldens via +`pnpm --filter runar-conformance run script-metrics`. + +### 2.1 Stack traffic is 23 % of the corpus — and 35–68 % of ordinary contracts + +| fixture | bytes | stack-shuffle share | +|---|---:|---:| +| `arithmetic` | 28 | **67.9 %** | +| `bounded-loop` | 42 | 57.1 % | +| `multisig` | 17 | 58.8 % | +| `if-without-else-multi-temp` | 226 | 55.3 % | +| `stateful-counter` | 1,875 | 35.9 % | +| `token-ft` | 3,154 | 36.1 % | +| `math-demo` | 17,348 | 35.7 % | + +### 2.2 The dead-slot hypothesis is **refuted** + +The obvious theory — dead values sink under live ones, so later accesses pay a deeper +`push(depth)`, and crossing depth 16 turns a 1-byte depth push into 2 — does not survive +measurement. Across all 387,749 `OP_PICK`/`OP_ROLL` sites in the corpus: + +``` +depth ≤ 16 : 387,092 (1-byte depth push) +depth > 16 : 657 (2-byte depth push) +deepest anywhere: 75 +``` + +Typical depths are 2–5. On `p256-wallet` every one of the 21,926 depth pushes is a single +byte. **Eager dead-slot retirement would cost 1–3 bytes per drop to save essentially nothing, +and is dropped from the design.** This is the main correction to the original plan. + +### 2.3 Where the shuffle bytes actually are + +`p256-wallet`: 173,967 shuffle bytes, of which only 43,852 are `PICK`/`ROLL` (op + depth +push). The remaining ~130 kB is bare one-byte shuffles — `OP_ROT`×30,406, `OP_SWAP`×27,342, +`OP_OVER`×23,417, `OP_DROP`×22,558, `OP_2DUP`×20,453 — and ~100 kB of that is the fixed +five-shuffle tail inside `cFieldMod`, i.e. crypto-macro output, not this scheduler. + +For the ordinary contracts the picture inverts: `arithmetic` spends 16 of 28 bytes on stack +access, split 8 bytes of depth pushes and 8 bytes of `PICK`/`ROLL`/`ROT`/`SWAP`. + +### 2.4 A measured headroom number + +`conformance/tests/arithmetic` is the only fixture whose bytes are produced *entirely* by +this pass. Source: + +```ts +const sum = a + b; const diff = a - b; const prod = a * b; const quot = a / b; +assert(sum + diff + prod + quot === this.target); +``` + +Emitted today (28 bytes, `00` = constructor placeholder): + +``` +OP_2DUP OP_ADD sum [a,b,sum] +OP_2 OP_PICK OP_2 OP_PICK OP_SUB diff [a,b,sum,diff] +OP_3 OP_PICK OP_3 OP_PICK OP_MUL prod [a,b,sum,diff,prod] +OP_4 OP_ROLL OP_4 OP_ROLL OP_DIV quot [sum,diff,prod,quot] +OP_3 OP_ROLL OP_3 OP_ROLL OP_ADD OP_ROT OP_ADD OP_SWAP OP_ADD + OP_NUMEQUAL +``` + +`a` and `b` are each materialized four times, and every materialization is deeper than the +last because each result is pushed on top of them. + +Hand-scheduled alternative (18 bytes) — operands stay in the top two slots, finished results +are parked on the alt stack: + +``` +OP_2DUP OP_ADD OP_TOALTSTACK sum -> alt +OP_2DUP OP_SUB OP_TOALTSTACK diff -> alt +OP_2DUP OP_MUL OP_TOALTSTACK prod -> alt +OP_DIV quot (consumes a, b) +OP_FROMALTSTACK OP_ADD + prod +OP_FROMALTSTACK OP_ADD + diff +OP_FROMALTSTACK OP_ADD + sum + OP_NUMEQUAL +``` + +**28 → 18 bytes, −36 %, and every byte saved is stack traffic.** Both scripts are executed +against the real `@bsv/sdk` interpreter over 14 (a, b) pairs including negatives, zero, and +the 16/17 script-number boundary, in +`packages/runar-testing/src/__tests__/scheduler-headroom.test.ts`; they accept and reject +identically. The number is measured, not estimated. + +The three effects that produced it, in order of contribution: + +1. **Result spilling to the alt stack.** Keeps the hot operands at depth 0/1, so every + subsequent access is `OP_2DUP` (1 byte) instead of `push d; OP_PICK; push d; OP_PICK` + (4 bytes). Worth 3 of the 4 materializations here. +2. **Adjacent-pair fusion.** `(a, b)` at depths 1 and 0 is `OP_2DUP`, not two picks. The + peephole has an `over,over → OP_2DUP` rule, but the lowerer emits `push;pick;push;pick`, + which no window can fuse. +3. **Consumption ordering.** Scheduling the one operator that *consumes* `a` and `b` + (`OP_DIV`) last removes the final pair of `OP_4 OP_ROLL`s entirely. + +--- + +## 3. Design + +### 3.1 Representation + +Extend the existing analysis rather than replacing it. `computeLastUses` already gives last +use; add, over the same scan: + +```ts +interface LivenessInfo { + lastUse: Map; // existing + useCount: Map; // total references (excluding array indirection) + nextUse: Map; // sorted binding indices that reference the name +} +``` + +`nextUse` is what makes "will this value be touched again soon, or not for a while?" a +question the scheduler can answer, and it is the input to the spill decision. It costs one +extra pass over the same bindings. + +### 3.2 Scheduling unit: the branch-free run + +The prototype operates only on a **maximal run of consecutive bindings containing no `if` +and no `loop`**. A run ends at any control-flow binding, and the stack is restored to a +canonical layout (everything on the main stack, alt stack empty) before that binding is +lowered. + +This is deliberate. `lowerIf`'s arm reconciliation, result-layout assertion and depth +balancing (§1.5) are the most delicate code in the backend, and every one of them reasons +about main-stack depth only. Confining spills to branch-free runs means **no arm ever begins +or ends with a non-empty alt stack**, so none of those invariants can be perturbed. It also +means the prototype cannot help inside a loop body — accepted for now; loops are unrolled, so +the runs *between* control flow are still scheduled. + +### 3.3 Byte-cost function + +`estimateScriptBytes` / `sizeOfStackOp` from `packages/runar-compiler/src/metrics/cost-model.ts` +— already implemented and asserted byte-exact against `06-emit.ts` over the whole corpus +(`__tests__/cost-model.test.ts`). The scheduler's local decisions use these derived costs: + +``` +accessCost(depth, consume) = 1 depth 0 (consume) — free + = 1 depth 0/1/2 via DUP/SWAP/OVER/ROT + = sizeOfPushValue(depth)+1 otherwise +pairAccessCost(d0, d1) = 1 (a,b) at depths 1,0 -> OP_2DUP + = accessCost(d0)+accessCost(d1) otherwise +spillCost = 2 TOALTSTACK + FROMALTSTACK +rematerializeCost(constant) = sizeOfPushValue(v) +``` + +### 3.4 Heuristic + +Greedy, single forward pass over a run. Not globally optimal, and deliberately so — the +brief asks for a simple greedy implementation first. + +For each binding `t := op(x, y)`: + +1. **Order the operands.** If `op` is commutative (`+ * === !== && || & | ^`, `min`, `max`), + order so the operand already nearer the top is materialized second. Ties keep source + order, so the default mode is unchanged. +2. **Fuse the pair.** If `(x, y)` sit at depths 1 and 0 and neither is consumed, emit + `OP_2DUP` instead of two accesses. Generalize to `OP_2OVER` for depths 3,2. +3. **Rematerialize instead of accessing.** If `x` is a `load_const` whose push encoding costs + ≤ `accessCost(depth(x), consume)`, re-push it and leave the resident copy alone. +4. **Spill the result.** After emitting the operation, if the result's `nextUse` is more than + `SPILL_HORIZON` bindings away *and* at least one still-live value sits below it, park it + with `OP_TOALTSTACK`. Restore in reverse spill order at the point of use. Spill only when + `spillCost < projected access savings`, computed from `nextUse` and the current depths. +5. **Restore before a run boundary.** Every spilled value is popped back before any `if`, + `loop`, or the end of the method. + +**As built, step 4 is stricter than this design anticipated.** Spilling is refused outright in +any scope that still has control flow ahead of it, because restoring immediately before an +`if` miscompiled a fixture — see §6. And step 1 is scored by running the candidate op +sequences through the real peephole rather than a byte formula, because the cheapest-looking +local choice is often one the peephole would have erased anyway (§6 again). + +The per-site heuristics are backed by a **method-level guard**: both schedules are lowered and +the cheaper one, measured after peephole with `estimateScriptBytes`, is kept. "The scheduler +never grows a method" is therefore a structural property, not a hope — which matters, because +the greedy heuristic cannot tell whether removing one slot actually moves an access across a +cost boundary (depths 0-2 are all one byte). + +### 3.5 Gating + +`schedulerMode: 'current' | 'liveness'` on `LoweringContext`, plumbed from +`CompileOptions.schedulerMode` (`packages/runar-compiler/src/index.ts:109`) and a CLI +`--stack-scheduler=` (`packages/runar-cli/src/bin.ts:39`, +`commands/compile.ts:14/84/177/252` — the `--disable-constant-folding` path is the template). + +Default is `'current'`, and every new behaviour is a no-op in that mode. This keeps the +72 goldens, `conformance/script-size-baseline.json`, the cross-tier hex parity gate and the +golden-provenance gate untouched while the experiment runs. + +--- + +## 4. Correctness invariants + +The scheduler may reorder *materialization*, never *evaluation*. Concretely: + +1. **Side-effect order is fixed.** Bindings are lowered in ANF order. Only the stack + operations that arrange operands may move. `hasSideEffect` (`optimizer/dce.ts:133`) names + the kinds that must never be reordered relative to each other. +2. **Operand order is preserved for non-commutative operators.** `-`, `/`, `%`, `<<`, `>>`, + `<`, `>`, `<=`, `>=`, `OP_SPLIT`, `OP_CAT` and every intrinsic keep source order. + Commutativity is asserted per-operator against the interpreter, not assumed: `OP_ADD` and + `OP_MUL` are commutative on script numbers; `OP_CAT` is not; `OP_BOOLAND`/`OP_BOOLOR` are + commutative but **not** short-circuit at this level, so reordering them cannot change + which side is evaluated (both already are). +3. **Alt stack is empty at every control-flow boundary** and at method exit. Asserted in the + lowerer, not just tested — a `LoweringContext` invariant check before each `if`/`loop` and + in `lowerMethod`. +4. **Main-stack depth at `OP_ENDIF` is unchanged.** The Layer B/C assertions (`:2417`, + `:2640`) stay in force; the prototype must not touch arm reconciliation at all. +5. **`maxStackDepth` may not exceed `MAX_STACK_DEPTH = 800`** (`:63`). Spilling *reduces* + main-stack depth, but the alt stack shares the interpreter's 1,000-element budget, so the + sum is what gets checked. +6. **No assertion is weakened.** The scheduler never removes an `assert` binding, never + changes which value an `OP_VERIFY` consumes, and never elides a normalization + (`OP_BIN2NUM`, `OP_NUM2BIN`, sign fixups) that a later consumer observes. + +--- + +## 5. Benchmark plan + +**Metric:** serialized locking-script bytes, from `estimateScriptBytes` (exact) and confirmed +against the emitted hex. + +**Command:** + +```bash +pnpm --filter runar-conformance run script-metrics -- --compare current,liveness +``` + +**Report, per fixture:** script bytes, `OP_PICK` / `OP_ROLL` / `OP_DUP` / `OP_SWAP` / +`OP_2DUP` / `OP_TOALTSTACK` counts, `maxStackDepth` delta. + +**Acceptance (from the brief, restated as pass/fail):** + +1. Semantically identical Script — proven, not assumed. `scheduler-equivalence.test.ts` + (modelled on `packages/runar-testing/src/oracle/fold-equivalence.ts`) compiles each source + under both modes and asserts identical accept/reject through `ScriptVM` plus agreement + with the mode-independent AST interpreter, over every witness in + `conformance/witnesses/`. Plus `conformance/fuzzer/index.ts --execute` with the toggle. +2. All existing VM / interpreter tests pass under both modes. +3. No material growth on ordinary fixtures. Fail the experiment if any fixture grows > 1 %. +4. A measurable win on at least one arithmetic-heavy fixture. **Target: > 10 %.** + +**Prior expectations, so the result could disappoint honestly:** + +| fixture class | expected | **measured** | +|---|---|---| +| `arithmetic` | −20 % to −36 % | **−35.7 %** (28 → 18 B) | +| `bounded-loop`, `boolean-logic` | −20 % to −36 % | −11.9 % (42 → 37 B) / 0 % | +| `math-demo`, `token-ft`, `function-patterns` | −3 % to −10 % | **−0.1 %** | +| stateful fixtures | −1 % to −4 % | −0.1 % to −0.2 % | +| EC / P-256 / P-384 | 0 % | 0 % | +| SLH-DSA / SHA-256 / BLAKE3 | 0 % | 0 % | + +`arithmetic` reached 18 bytes — the hand-derived optimum in §2.4 — which the scheduler found +on its own. The mid-size prediction was wrong by an order of magnitude: those contracts do have +a 35 % stack-shuffle share, but almost all of it is sighash and state-serialization macro +output, not ANF chains the scheduler can reach. Their ANF is mostly bindings consumed by the +very next binding, where there is nothing to spill. + +So the honest conclusion is the one the plan named as the disappointing case: **the generic +scheduler is worth having for small arithmetic contracts and little else**, and the remaining +shuffle budget belongs to the macro emitters. Corpus-wide it moves 34 of 72 fixtures and +−0.0 % of total bytes. It is kept as a gated mode, not proposed for a 7-tier port. + +--- + +## 6. What went wrong, and what caught it + +### A miscompile, caught by the witness corpus + +The first working scheduler **miscompiled `if-without-else-multi-temp`**: the script ran to +completion, left a truthy top-of-stack, and **accepted a witness the shipping compiler +rejects**. Byte counts, the goldens for every other fixture, and 4,099 compiler unit tests all +passed while that was true. + +`conformance/witnesses/` replayed through `runDifferentialExecution` caught it — deployed +script versus the ANF interpreter, on witnesses the repo had already committed to, with at +least one accept and one reject per fixture. Two of 86 cases failed. + +Cause: restoring spilled values immediately before an `if` leaves the parent stack in a shape +`lowerIf`'s arm reconciliation, declared-result trim and Layer B/C depth invariants were not +written for. The fix is the precondition in §3.4 — refuse to spill in a scope with control +flow ahead of it — rather than an attempt to make the two agree. + +### Two things worth remembering + +**A passing bisect can be vacuous.** Turning off commutative reordering made the failure +disappear, which looked like an acquittal for spilling. It was not: with reordering off, the +method-level cost guard simply preferred the baseline schedule, so no spilling happened at +all. Only after confirming the variant still changed the emitted bytes did the second bisect +mean anything. + +**The cost model had to be peephole-aware.** Two consumed operands at depths 1 and 0 emit +`OP_SWAP OP_SWAP`, which the `swap-swap` rule deletes outright — free — while the +"cheaper-looking" reversed order emits one real `OP_SWAP` and costs a byte. Scoring candidate +op sequences through `optimizeStackIR` before comparing them took `arithmetic` from 24 bytes +to 18. diff --git a/packages/runar-cli/src/__tests__/experimental-flags.test.ts b/packages/runar-cli/src/__tests__/experimental-flags.test.ts new file mode 100644 index 000000000..2626d233f --- /dev/null +++ b/packages/runar-cli/src/__tests__/experimental-flags.test.ts @@ -0,0 +1,131 @@ +// --------------------------------------------------------------------------- +// Tests for the experimental size-optimizer flags on the TS CLI: +// --stack-scheduler +// --ec-constant-pool +// +// Both change emitted bytes when enabled and MUST be inert when absent, since +// the checked-in goldens, `conformance/script-size-baseline.json` and the +// cross-tier hex parity gate all assume the default path. +// +// See docs/experiments/script-size-optimizer-results.md. +// --------------------------------------------------------------------------- + +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; + +/** The smallest contract whose bytes come entirely from the generic scheduler. */ +const ARITHMETIC = ` +import { SmartContract, assert } from 'runar-lang'; + +class Arithmetic extends SmartContract { + readonly target: bigint; + constructor(target: bigint) { + super(target); + this.target = target; + } + public verify(a: bigint, b: bigint): void { + const sum: bigint = a + b; + const diff: bigint = a - b; + const prod: bigint = a * b; + const quot: bigint = a / b; + const result: bigint = sum + diff + prod + quot; + assert(result === this.target); + } +} +`; + +let workDir: string; + +beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runar-cli-exp-')); +}); + +afterAll(() => { + if (workDir) fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe('experimental size-optimizer flags', () => { + let compileCommand: typeof import('../commands/compile.js').compileCommand; + + beforeAll(async () => { + const sourceEntry = path.resolve(process.cwd(), 'packages/runar-compiler/src/index.ts'); + if (fs.existsSync(sourceEntry)) { + const { pathToFileURL } = await import('node:url'); + await import(pathToFileURL(sourceEntry).href); + } else { + await import('runar-compiler'); + } + }, 60_000); + + beforeEach(async () => { + ({ compileCommand } = await import('../commands/compile.js')); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + /** Compile ARITHMETIC with the given options and return the printed hex. */ + async function hexWith(options: Record, tag: string): Promise { + const srcPath = path.join(workDir, `Arithmetic-${tag}.runar.ts`); + fs.writeFileSync(srcPath, ARITHMETIC); + const outDir = path.join(workDir, `out-${tag}`); + fs.mkdirSync(outDir, { recursive: true }); + + const writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const originalExitCode = process.exitCode; + + await compileCommand([srcPath], { output: outDir, hex: true, ...options }); + + const printed = writeSpy.mock.calls.map(c => String(c[0])).join('').trim(); + process.exitCode = originalExitCode; + writeSpy.mockRestore(); + return printed; + } + + it('is inert when no experimental flag is passed', async () => { + const bare = await hexWith({}, 'bare'); + const explicit = await hexWith({ stackScheduler: 'current' }, 'explicit'); + expect(bare).toBe(explicit); + // The shipping schedule for this contract; also the checked-in golden for + // conformance/tests/arithmetic modulo the constructor placeholder. + expect(bare).toBe('6e9352795279945379537995547a547a96537a537a937b937c93009c'); + expect(bare.length / 2).toBe(28); + }); + + it('--stack-scheduler liveness reschedules and shrinks', async () => { + const out = await hexWith({ stackScheduler: 'liveness' }, 'liveness'); + expect(out).toBe('6e936b6e946b6e956b966c6c6c939393009c'); + expect(out.length / 2).toBe(18); + }); + + it('rejects an unknown scheduler mode instead of silently using the default', async () => { + // A benchmark run that quietly measured the shipping compiler while + // reporting an experiment is worse than a crash. + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const originalExitCode = process.exitCode; + + const srcPath = path.join(workDir, 'Arithmetic-bad.runar.ts'); + fs.writeFileSync(srcPath, ARITHMETIC); + const outDir = path.join(workDir, 'out-bad'); + fs.mkdirSync(outDir, { recursive: true }); + + await compileCommand([srcPath], { output: outDir, hex: true, stackScheduler: 'bogus' }); + + const errors = errSpy.mock.calls.map(c => String(c[0])).join('\n'); + expect(errors).toMatch(/unknown mode 'bogus'/); + process.exitCode = originalExitCode; + }); + + it('--ec-constant-pool is inert on a contract with no EC operations', async () => { + const bare = await hexWith({}, 'nopool'); + const pooled = await hexWith({ ecConstantPool: true }, 'pool'); + expect(pooled).toBe(bare); + }); +}); diff --git a/packages/runar-cli/src/bin.ts b/packages/runar-cli/src/bin.ts index 1a4ade66d..506f1a645 100644 --- a/packages/runar-cli/src/bin.ts +++ b/packages/runar-cli/src/bin.ts @@ -43,6 +43,8 @@ program .option('--ir', 'include IR in artifact') .option('--asm', 'print ASM to stdout') .option('--disable-constant-folding', 'disable ANF constant folding pass') + .option('--ec-constant-pool', 'EXPERIMENTAL: pool repeated EC curve constants (changes emitted bytes)') + .option('--stack-scheduler ', 'EXPERIMENTAL: operand scheduling — current|liveness', 'current') .option('--from-ir ', 'compile from an ANF IR JSON file (skips parse/validate/typecheck/anf-lower)') .option('--hex', 'print only the script hex to stdout (no artifact JSON)') .option('--parse-only', 'stop after parse + validate; print "parser ok" on success (requires source input)') diff --git a/packages/runar-cli/src/commands/compile.ts b/packages/runar-cli/src/commands/compile.ts index 74afbd29c..97a710511 100644 --- a/packages/runar-cli/src/commands/compile.ts +++ b/packages/runar-cli/src/commands/compile.ts @@ -12,6 +12,8 @@ interface CompileOptions { ir?: boolean; asm?: boolean; disableConstantFolding?: boolean; + ecConstantPool?: boolean; + stackScheduler?: string; fromIr?: string; hex?: boolean; parseOnly?: boolean; @@ -72,6 +74,19 @@ function repoRelativeFileName(absSourcePath: string): string { * 3. Write the resulting artifact JSON to the output directory. * 4. Optionally print the ASM to stdout. */ +/** + * Validate `--stack-scheduler`. An unrecognised mode is an error rather than a + * silent fall back to the default: a benchmark run that quietly measured the + * shipping compiler while reporting an experiment would be worse than a crash. + */ +function schedulerMode(options: { stackScheduler?: string }): 'current' | 'liveness' { + const mode = options.stackScheduler ?? 'current'; + if (mode !== 'current' && mode !== 'liveness') { + throw new Error(`--stack-scheduler: unknown mode '${mode}' (expected current|liveness)`); + } + return mode; +} + export async function compileCommand( files: string[], options: CompileOptions, @@ -81,10 +96,10 @@ export async function compileCommand( // Dynamically import the compiler to avoid hard failures if it's not // yet fully built (the compiler package may still be under development). - type CompileFn = (source: string, options?: { fileName?: string; disableConstantFolding?: boolean; parseOnly?: boolean }) => unknown; + type CompileFn = (source: string, options?: { fileName?: string; disableConstantFolding?: boolean; ecConstantPool?: boolean; schedulerMode?: 'current' | 'liveness'; parseOnly?: boolean }) => unknown; type CompileFromANFFn = ( program: unknown, - options?: { disableConstantFolding?: boolean }, + options?: { disableConstantFolding?: boolean; ecConstantPool?: boolean; schedulerMode?: 'current' | 'liveness' }, ) => { scriptHex: string; scriptAsm: string }; type LoadANFFn = (json: string) => unknown; @@ -174,7 +189,11 @@ export async function compileCommand( let result: { scriptHex: string; scriptAsm: string }; try { - result = compileFromANF(program, { disableConstantFolding: options.disableConstantFolding }); + result = compileFromANF(program, { + disableConstantFolding: options.disableConstantFolding, + ecConstantPool: options.ecConstantPool, + schedulerMode: schedulerMode(options), + }); } catch (err) { console.error(` Compilation error: ${(err as Error).message}`); process.exitCode = 1; @@ -250,6 +269,8 @@ export async function compileCommand( compileResult = compile(source, { fileName: resolvedPath, disableConstantFolding: options.disableConstantFolding, + ecConstantPool: options.ecConstantPool, + schedulerMode: schedulerMode(options), parseOnly: options.parseOnly, }) as CompileResultLike; } catch (err) { diff --git a/packages/runar-compiler/src/__tests__/cost-model.test.ts b/packages/runar-compiler/src/__tests__/cost-model.test.ts new file mode 100644 index 000000000..ae24e1b3a --- /dev/null +++ b/packages/runar-compiler/src/__tests__/cost-model.test.ts @@ -0,0 +1,198 @@ +/** + * Script-byte cost model — exactness tests. + * + * The cost model exists so optimizer passes can compare two candidate + * lowerings by the metric that actually matters (serialized locking-script + * bytes) BEFORE emitting either. That is only useful if the estimate is not + * an estimate at all: the contract asserted here is + * + * estimateScriptBytes(ops) === emitMethod({ ops, ... }).scriptHex.length / 2 + * + * for every op sequence the compiler can produce. The sweep below runs that + * equality over every conformance fixture, so the model is a CHECKED MIRROR + * of `06-emit.ts` rather than a second, drifting opinion about encoding. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'fs'; +import { join, resolve } from 'path'; +import { parse } from '../passes/01-parse.js'; +import { lowerToANF } from '../passes/04-anf-lower.js'; +import { lowerToStack } from '../passes/05-stack-lower.js'; +import { emitMethod } from '../passes/06-emit.js'; +import { optimizeStackIR } from '../optimizer/peephole.js'; +import { sizeOfStackOp, estimateScriptBytes } from '../metrics/cost-model.js'; +import type { StackOp } from '../ir/index.js'; + +const CONFORMANCE_DIR = join(__dirname, '..', '..', '..', '..', 'conformance', 'tests'); + +/** Emit a bare op list through the real emitter and return its byte length. */ +function emittedBytes(ops: StackOp[]): number { + const result = emitMethod({ name: 'probe', ops, maxStackDepth: 0 }); + return result.scriptHex.length / 2; +} + +/** Assert the model agrees with the emitter for one op sequence. */ +function expectExact(ops: StackOp[]): void { + expect(estimateScriptBytes(ops)).toBe(emittedBytes(ops)); +} + +// --------------------------------------------------------------------------- +// Per-op-kind units +// --------------------------------------------------------------------------- + +describe('sizeOfStackOp', () => { + it('costs a named opcode at one byte', () => { + expect(sizeOfStackOp({ op: 'opcode', code: 'OP_ADD' })).toBe(1); + }); + + it('throws on an unknown opcode rather than silently costing zero', () => { + expect(() => sizeOfStackOp({ op: 'opcode', code: 'OP_NOT_A_REAL_OPCODE' })).toThrow( + /OP_NOT_A_REAL_OPCODE/, + ); + }); + + it.each([ + ['dup'], ['swap'], ['drop'], ['nip'], ['over'], ['rot'], ['tuck'], + ] as const)('costs the nullary shuffle %s at one byte', (op) => { + expect(sizeOfStackOp({ op } as StackOp)).toBe(1); + }); + + it('costs pick/roll at one byte — the depth push is a separate op', () => { + // bringToTop emits `push(depth)` and `pick{depth}` as TWO ops; counting + // the depth inside the pick would double-charge it. + expect(sizeOfStackOp({ op: 'pick', depth: 40 })).toBe(1); + expect(sizeOfStackOp({ op: 'roll', depth: 40 })).toBe(1); + }); + + it('costs placeholder and codesep-index at one byte each', () => { + expect(sizeOfStackOp({ op: 'placeholder', paramIndex: 0, paramName: 'x' })).toBe(1); + expect(sizeOfStackOp({ op: 'push_codesep_index' })).toBe(1); + }); + + it('costs raw_bytes at its verbatim length', () => { + const bytes = new Uint8Array([0x51, 0x52, 0x93]); + expect(sizeOfStackOp({ op: 'raw_bytes', bytes, in_arity: 0, out_arity: 1 })).toBe(3); + }); + + describe('push encoding', () => { + it.each([ + [0n, 1], // OP_0 + [1n, 1], // OP_1 + [16n, 1], // OP_16 + [-1n, 1], // OP_1NEGATE + [17n, 2], // len prefix + 1 byte + [127n, 2], + [128n, 3], // sign byte forces 2 data bytes + [-128n, 3], + [65535n, 4], + ])('costs push(%s) at %i bytes', (value, want) => { + expect(sizeOfStackOp({ op: 'push', value })).toBe(want); + }); + + it('costs the P-256 field prime push at 34 bytes', () => { + // 32 magnitude bytes + 1 sign byte + 1 length prefix. This single push + // accounts for 680,850 of p256-wallet's 958,792 bytes. + const p = 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn; + expect(sizeOfStackOp({ op: 'push', value: p })).toBe(34); + }); + + it('costs byte-array pushes across the PUSHDATA boundaries', () => { + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(0) })).toBe(1); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(1).fill(0xaa) })).toBe(2); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(75).fill(0xaa) })).toBe(76); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(76).fill(0xaa) })).toBe(78); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(255).fill(0xaa) })).toBe(257); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(256).fill(0xaa) })).toBe(259); + }); + + it('costs boolean pushes like the emitter encodes them', () => { + expectExact([{ op: 'push', value: true }]); + expectExact([{ op: 'push', value: false }]); + }); + }); + + describe('if', () => { + it('costs OP_IF + body + OP_ENDIF when there is no else arm', () => { + const op: StackOp = { op: 'if', then: [{ op: 'opcode', code: 'OP_ADD' }] }; + expect(sizeOfStackOp(op)).toBe(3); + expectExact([op]); + }); + + it('costs OP_IF + then + OP_ELSE + else + OP_ENDIF', () => { + const op: StackOp = { + op: 'if', + then: [{ op: 'opcode', code: 'OP_ADD' }], + else: [{ op: 'push', value: 0n }], + }; + expect(sizeOfStackOp(op)).toBe(5); + expectExact([op]); + }); + + it('recurses into nested arms', () => { + const inner: StackOp = { op: 'if', then: [{ op: 'opcode', code: 'OP_ADD' }] }; + const outer: StackOp = { op: 'if', then: [inner], else: [inner] }; + // outer IF(1) + inner(3) + ELSE(1) + inner(3) + ENDIF(1) + expect(sizeOfStackOp(outer)).toBe(9); + expectExact([outer]); + }); + + it('omits OP_ELSE for an empty else arm, matching emitIf', () => { + const op: StackOp = { op: 'if', then: [{ op: 'opcode', code: 'OP_ADD' }], else: [] }; + expect(sizeOfStackOp(op)).toBe(3); + expectExact([op]); + }); + }); +}); + +// --------------------------------------------------------------------------- +// The real contract: exact agreement with the emitter, over every fixture +// --------------------------------------------------------------------------- + +interface SourceConfig { + sources?: Record; +} + +function tsSourceFor(fixture: string): string | null { + const configFile = join(CONFORMANCE_DIR, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf-8')) as SourceConfig; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(CONFORMANCE_DIR, fixture, rel); + if (!existsSync(abs)) throw new Error(`source.json points at a missing file: ${abs}`); + return abs; +} + +const FIXTURES = readdirSync(CONFORMANCE_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(name => tsSourceFor(name) !== null) + .sort(); + +describe('estimateScriptBytes matches the emitter exactly', () => { + it('found the conformance corpus', () => { + expect(FIXTURES.length).toBeGreaterThan(50); + }); + + it.each(FIXTURES)('%s', (fixture) => { + const path = tsSourceFor(fixture)!; + const source = readFileSync(path, 'utf-8'); + const parsed = parse(source, path); + if (!parsed.contract) { + throw new Error(`parse failed for ${fixture}: ${parsed.errors.map(e => e.message).join(', ')}`); + } + const stack = lowerToStack(lowerToANF(parsed.contract)); + + for (const method of stack.methods) { + // Both before and after peephole: the model must be exact on any op + // sequence the pipeline can hand the emitter, not just the final one. + expect(estimateScriptBytes(method.ops)).toBe(emitMethod(method).scriptHex.length / 2); + + const optimized = { ...method, ops: optimizeStackIR(method.ops) }; + expect(estimateScriptBytes(optimized.ops)).toBe( + emitMethod(optimized).scriptHex.length / 2, + ); + } + }); +}); diff --git a/packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts b/packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts new file mode 100644 index 000000000..4654ccfbe --- /dev/null +++ b/packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts @@ -0,0 +1,146 @@ +/** + * EC constant pooling — size and default-invariance. + * + * `cFieldMod` / `fieldMod` push the curve's field prime inline at EVERY + * modular reduction. On `conformance/tests/p256-wallet` that is 20,025 pushes + * of a 34-byte literal — 680,850 of the fixture's 958,792 bytes, 71 %. The + * prime is a compile-time constant; parking one copy in a stack slot and + * copying it with `push d; OP_PICK` costs 2-3 bytes instead of 34. + * + * This file pins two things: + * 1. with pooling OFF the emitters are byte-identical to what ships today + * (so no golden, baseline, or cross-tier parity gate can move), and + * 2. with pooling ON the scripts actually shrink, by the amount the + * arithmetic predicts rather than by "some". + * + * Semantic equivalence is proved separately, against OpenSSL signatures on the + * real interpreter, in + * `packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts`. + */ + +import { describe, it, expect } from 'vitest'; +import { + emitVerifyECDSA_P256, emitVerifyECDSA_P384, + emitP256Mul, emitP256MulGen, emitP256Add, emitP256OnCurve, + emitP384Mul, emitP384Add, +} from '../passes/p256-p384-codegen.js'; +import { + emitEcAdd, emitEcMul, emitEcMulGen, +} from '../passes/ec-codegen.js'; +import { emitMethod } from '../passes/06-emit.js'; +import { estimateScriptBytes } from '../metrics/cost-model.js'; +import { analyzeScriptHex } from '../metrics/script-metrics.js'; +import { encodeScriptNumber } from '../passes/push-encoding.js'; +import type { StackOp } from '../ir/index.js'; +import type { EcCodegenOptions } from '../passes/ec-codegen.js'; + +type Emitter = (emit: (op: StackOp) => void, opts?: EcCodegenOptions) => void; + +function opsOf(emitter: Emitter, opts?: EcCodegenOptions): StackOp[] { + const ops: StackOp[] = []; + emitter(op => ops.push(op), opts); + return ops; +} + +function hexOf(ops: StackOp[]): string { + return emitMethod({ name: 'probe', ops, maxStackDepth: 0 }).scriptHex; +} + +function bytesOf(emitter: Emitter, opts?: EcCodegenOptions): number { + return estimateScriptBytes(opsOf(emitter, opts)); +} + +/** + * Net stack effect of an op sequence, counting only the ops whose effect is + * unambiguous from the Stack IR alone (pushes and pops). Opcodes are opaque + * here, so this is a same-shape comparison between two variants of the SAME + * emitter, not an absolute depth model — which is all the pool needs to prove: + * every slot it pushes, it releases. + */ +function netStackEffect(ops: StackOp[]): number { + let net = 0; + const walk = (list: StackOp[]): void => { + for (const op of list) { + if (op.op === 'push' || op.op === 'dup' || op.op === 'over' || op.op === 'tuck' + || op.op === 'placeholder' || op.op === 'push_codesep_index') net++; + else if (op.op === 'drop' || op.op === 'nip') net--; + // A pick/roll is always preceded by a `push` of the depth, already + // counted above. OP_PICK consumes that depth and pushes a copy (net 0); + // OP_ROLL consumes it and relocates an existing item (net -1). + else if (op.op === 'roll') net--; + else if (op.op === 'if') { walk(op.then); if (op.else) walk(op.else); } + } + }; + walk(ops); + return net; +} + +/** Every emitter that should benefit, with its shipping byte count. */ +const EMITTERS: Array<[string, Emitter, number]> = [ + ['emitVerifyECDSA_P256', emitVerifyECDSA_P256, 974024], + ['emitVerifyECDSA_P384', emitVerifyECDSA_P384, 1987394], + ['emitP256Mul', emitP256Mul, 459746], + ['emitP256MulGen', emitP256MulGen, 459812], + ['emitP256Add', emitP256Add, 19906], + ['emitP384Mul', emitP384Mul, 927350], + ['emitP384Add', emitP384Add, 46710], + ['emitEcAdd', emitEcAdd, 25426], + ['emitEcMul', emitEcMul, 428676], + ['emitEcMulGen', emitEcMulGen, 428742], +]; + +describe('pooling OFF is the shipping default', () => { + it.each(EMITTERS)('%s emits its documented byte count', (_name, emitter, want) => { + expect(bytesOf(emitter)).toBe(want); + }); + + it.each(EMITTERS)('%s is byte-identical with an explicit constantPool:false', (_n, emitter) => { + expect(hexOf(opsOf(emitter))).toBe(hexOf(opsOf(emitter, { constantPool: false }))); + }); + + it('an empty options object changes nothing', () => { + expect(hexOf(opsOf(emitP256OnCurve))).toBe(hexOf(opsOf(emitP256OnCurve, {}))); + }); +}); + +describe('pooling ON removes the repeated prime pushes', () => { + it('collapses the P-256 field prime from 34 bytes a push to a pick', () => { + const before = analyzeScriptHex(hexOf(opsOf(emitVerifyECDSA_P256))); + const after = analyzeScriptHex(hexOf(opsOf(emitVerifyECDSA_P256, { constantPool: true }))); + + const p = Buffer.from( + encodeScriptNumber(0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn), + ).toString('hex'); + const beforeP = before.constants.find(c => c.hex === p)!; + expect(beforeP.count).toBeGreaterThan(15000); + + const afterP = after.constants.find(c => c.hex === p); + // At most a handful survive: the pool slot itself, plus any site that + // genuinely cannot see the slot. + expect(afterP?.count ?? 0).toBeLessThan(10); + }); + + it.each(EMITTERS)('%s shrinks by more than half', (_name, emitter, before) => { + const after = bytesOf(emitter, { constantPool: true }); + expect(after).toBeLessThan(before * 0.5); + }); + + it('brings verifyECDSA_P256 close to the arithmetic prediction', () => { + // 20,025-ish reductions x ~31 bytes saved each on the p256-wallet fixture; + // the bare emitter carries the same reduction count. Predicted landing + // zone is ~330 kB. Allow slack, but fail if it lands nowhere near. + const after = bytesOf(emitVerifyECDSA_P256, { constantPool: true }); + expect(after).toBeGreaterThan(200_000); + expect(after).toBeLessThan(420_000); + }); + + it('adds at most two resident slots', () => { + // The pool is two extra stack items per tracker (p and n). Real max-depth + // is measured on the interpreter in the equivalence test; here we only pin + // that the emitter still balances — pool slots pushed are pool slots + // released, so the net stack effect is unchanged. + const off = opsOf(emitP256Add); + const on = opsOf(emitP256Add, { constantPool: true }); + expect(netStackEffect(on)).toBe(netStackEffect(off)); + }); +}); diff --git a/packages/runar-compiler/src/__tests__/golden-invariance.test.ts b/packages/runar-compiler/src/__tests__/golden-invariance.test.ts new file mode 100644 index 000000000..afdc89fe0 --- /dev/null +++ b/packages/runar-compiler/src/__tests__/golden-invariance.test.ts @@ -0,0 +1,58 @@ +/** + * Default output is byte-identical to the checked-in goldens. + * + * The size experiments in `docs/experiments/` add opt-in flags that change + * emitted bytes. This is the guard that says the DEFAULT path did not move: + * every fixture that ships a `.runar.ts`, compiled with the same options the + * goldens were stamped under (fold-OFF), must reproduce + * `conformance/tests//expected-script.hex` exactly. + * + * `conformance/runner/runner.ts` checks this across all seven tiers in CI, but + * that needs six native toolchains built. This is the TS-tier-only version that + * runs anywhere in seconds-to-minutes, so an experiment can be shown to be + * byte-neutral without a full conformance run. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'fs'; +import { join, resolve } from 'path'; +import { compile } from '../index.js'; + +const CONFORMANCE_DIR = join(__dirname, '..', '..', '..', '..', 'conformance', 'tests'); + +function tsSourceFor(fixture: string): string | null { + const configFile = join(CONFORMANCE_DIR, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf-8')) as { sources?: Record }; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(CONFORMANCE_DIR, fixture, rel); + if (!existsSync(abs)) throw new Error(`source.json points at a missing file: ${abs}`); + return abs; +} + +const FIXTURES = readdirSync(CONFORMANCE_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(name => tsSourceFor(name) !== null + && existsSync(join(CONFORMANCE_DIR, name, 'expected-script.hex'))) + .sort(); + +describe('default compilation reproduces the goldens', () => { + it('found the corpus', () => { + expect(FIXTURES.length).toBeGreaterThan(50); + }); + + it.each(FIXTURES)('%s', (fixture) => { + const path = tsSourceFor(fixture)!; + const golden = readFileSync(join(CONFORMANCE_DIR, fixture, 'expected-script.hex'), 'utf-8') + .replace(/\s+/g, ''); + // Goldens are stamped fold-OFF (CLAUDE.md, CONTRIBUTING.md). + const result = compile(readFileSync(path, 'utf-8'), { + fileName: path, + disableConstantFolding: true, + }); + expect(result.success, result.diagnostics.map(d => d.message).join('; ')).toBe(true); + expect(result.scriptHex).toBe(golden); + }); +}); diff --git a/packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts b/packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts new file mode 100644 index 000000000..00751b8c4 --- /dev/null +++ b/packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts @@ -0,0 +1,111 @@ +/** + * Liveness-aware stack scheduling — size and default-invariance. + * + * ANF names every intermediate, and the current lowering pushes each result on + * top of the operands that produced it. In an arithmetic chain that reads the + * same two values repeatedly, every result buries them one slot deeper, so the + * next access costs a `push d; OP_PICK` pair instead of a 1-byte `OP_2DUP`. + * `conformance/tests/arithmetic` spends 16 of its 28 bytes exactly that way. + * + * The `liveness` scheduler parks a result on the alt stack when the next + * binding does not want it, keeping the hot operands at depth 0/1, and + * restores the whole spill group in one go before the first binding that + * needs any of it. Restoring en masse puts the values back in production + * order (first-spilled on top), which is the order an ANF accumulation chain + * consumes them in. + * + * Pinned here: (1) `current` mode is byte-identical to what ships, and + * (2) `liveness` mode actually shrinks the arithmetic-heavy fixtures. + * Semantic equivalence is proved on the real interpreter in + * `packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts`. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'fs'; +import { join, resolve } from 'path'; +import { compile } from '../index.js'; +import { analyzeScriptHex } from '../metrics/script-metrics.js'; + +const CONFORMANCE_DIR = join(__dirname, '..', '..', '..', '..', 'conformance', 'tests'); + +function tsSourceFor(fixture: string): string | null { + const configFile = join(CONFORMANCE_DIR, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf-8')) as { sources?: Record }; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(CONFORMANCE_DIR, fixture, rel); + return existsSync(abs) ? abs : null; +} + +function hexFor(fixture: string, liveness: boolean): string { + const path = tsSourceFor(fixture)!; + const result = compile(readFileSync(path, 'utf-8'), { + fileName: path, + disableConstantFolding: true, + ...(liveness ? { schedulerMode: 'liveness' as const } : {}), + }); + expect(result.success, result.diagnostics.map(d => d.message).join('; ')).toBe(true); + return result.scriptHex!; +} + +const ALL_FIXTURES = readdirSync(CONFORMANCE_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(name => tsSourceFor(name) !== null + && existsSync(join(CONFORMANCE_DIR, name, 'expected-script.hex'))) + .sort(); + +describe('scheduler mode "current" is the shipping default', () => { + it.each(ALL_FIXTURES)('%s matches its golden', (fixture) => { + const golden = readFileSync(join(CONFORMANCE_DIR, fixture, 'expected-script.hex'), 'utf-8') + .replace(/\s+/g, ''); + expect(hexFor(fixture, false)).toBe(golden); + }); +}); + +describe('scheduler mode "liveness"', () => { + it('never grows a fixture', () => { + // A scheduler that trades bytes for bytes is not an optimization. Every + // spill decision goes through the cost model, so growth is a bug. + const grew: string[] = []; + for (const fixture of ALL_FIXTURES) { + const before = hexFor(fixture, false).length / 2; + const after = hexFor(fixture, true).length / 2; + if (after > before) grew.push(`${fixture}: ${before} -> ${after}`); + } + expect(grew).toEqual([]); + }); + + it('shrinks the arithmetic fixture by more than 10 %', () => { + const before = hexFor('arithmetic', false).length / 2; + const after = hexFor('arithmetic', true).length / 2; + expect(before).toBe(28); + expect(1 - after / before).toBeGreaterThan(0.1); + }); + + it('replaces PICK/ROLL traffic with alt-stack round trips on arithmetic', () => { + const before = analyzeScriptHex(hexFor('arithmetic', false)); + const after = analyzeScriptHex(hexFor('arithmetic', true)); + const shuffle = (m: typeof before) => m.categories['stack-shuffle']; + expect(shuffle(after)).toBeLessThan(shuffle(before)); + expect(after.opcodes['OP_PICK'] ?? 0).toBeLessThan(before.opcodes['OP_PICK'] ?? 0); + expect(after.opcodes['OP_TOALTSTACK'] ?? 0).toBeGreaterThan(0); + }); + + it('balances every spill it introduces', () => { + // A static count is NOT a balance proof on its own: `sha256-finalize` + // already emits 896 OP_TOALTSTACK against 897 OP_FROMALTSTACK, because + // one arm of an `if` pushes to the alt stack and the other does not, and + // both arms are counted. So the invariant is a DELTA one: whatever the + // scheduler adds must be added in pairs. + for (const fixture of ALL_FIXTURES) { + const before = analyzeScriptHex(hexFor(fixture, false)); + const after = analyzeScriptHex(hexFor(fixture, true)); + const to = (after.opcodes['OP_TOALTSTACK'] ?? 0) - (before.opcodes['OP_TOALTSTACK'] ?? 0); + const from = (after.opcodes['OP_FROMALTSTACK'] ?? 0) - (before.opcodes['OP_FROMALTSTACK'] ?? 0); + expect(to, `${fixture}: unbalanced spill traffic`).toBe(from); + expect(to, `${fixture}: negative spill count`).toBeGreaterThanOrEqual(0); + } + }); +}); diff --git a/packages/runar-compiler/src/__tests__/script-metrics.test.ts b/packages/runar-compiler/src/__tests__/script-metrics.test.ts new file mode 100644 index 000000000..d17f406d3 --- /dev/null +++ b/packages/runar-compiler/src/__tests__/script-metrics.test.ts @@ -0,0 +1,152 @@ +/** + * Script-size instrumentation — tests. + * + * `analyzeScriptHex` answers "where did the bytes go?" for a serialized + * locking script. It exists because the interesting question about a 958 kB + * P-256 verifier is not how many opcodes it has, but which KIND of byte + * dominates — and the answer (73 % literal pushes of one 33-byte constant) + * is invisible from an opcode histogram alone. + * + * The classifier's one subtle rule: a push immediately consumed by OP_PICK / + * OP_ROLL is stack-access cost, not a constant. Charging it to `const-push` + * would blame the wrong optimizer for a third of the shuffle traffic. + */ + +import { describe, it, expect } from 'vitest'; +import { analyzeScriptHex, stackOpMetrics } from '../metrics/script-metrics.js'; +import { emitMethod } from '../passes/06-emit.js'; +import type { StackOp } from '../ir/index.js'; + +function hexOf(ops: StackOp[]): string { + return emitMethod({ name: 'probe', ops, maxStackDepth: 0 }).scriptHex; +} + +describe('analyzeScriptHex', () => { + it('accounts for every byte exactly once', () => { + const hex = hexOf([ + { op: 'push', value: 0xdeadbeefn }, + { op: 'dup' }, + { op: 'opcode', code: 'OP_ADD' }, + { op: 'push', value: new Uint8Array(80).fill(0xaa) }, + { op: 'drop' }, + ]); + const m = analyzeScriptHex(hex); + const summed = Object.values(m.categories).reduce((a, b) => a + b, 0); + expect(m.scriptBytes).toBe(hex.length / 2); + expect(summed).toBe(m.scriptBytes); + }); + + it('separates small-int pushes from data pushes', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: 5n }, // OP_5, 1 byte + { op: 'push', value: 0n }, // OP_0, 1 byte + { op: 'push', value: 1000n }, // 1 len + 2 data + ])); + expect(m.categories['small-int-push']).toBe(2); + expect(m.categories['const-push']).toBe(3); + }); + + it('charges a PICK/ROLL depth push to stack-shuffle, not const-push', () => { + // This is how `bringToTop` materializes a deep operand: push(depth) then + // OP_PICK. Both bytes are stack-access cost. + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: 40n }, // 1 len + 1 data = 2 bytes + { op: 'pick', depth: 40 }, // 1 byte + ])); + expect(m.categories['stack-shuffle']).toBe(3); + expect(m.categories['const-push']).toBe(0); + expect(m.categories['small-int-push']).toBe(0); + }); + + it('charges a small-int depth push to stack-shuffle too', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: 3n }, // OP_3, 1 byte + { op: 'roll', depth: 3 }, // 1 byte + ])); + expect(m.categories['stack-shuffle']).toBe(2); + expect(m.categories['small-int-push']).toBe(0); + }); + + it('classifies arithmetic and control separately from shuffles', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'opcode', code: 'OP_ADD' }, + { op: 'opcode', code: 'OP_MOD' }, + { op: 'swap' }, + { op: 'if', then: [{ op: 'opcode', code: 'OP_MUL' }] }, + { op: 'opcode', code: 'OP_VERIFY' }, + ])); + expect(m.categories['arithmetic']).toBe(3); // ADD, MOD, MUL + expect(m.categories['stack-shuffle']).toBe(1); // SWAP + expect(m.categories['control']).toBe(3); // IF, ENDIF, VERIFY + }); + + it('counts repeated data constants and their total byte cost', () => { + const p = 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn; + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: p }, + { op: 'opcode', code: 'OP_MOD' }, + { op: 'push', value: p }, + { op: 'opcode', code: 'OP_MOD' }, + { op: 'push', value: p }, + ])); + const top = m.constants[0]!; + expect(top.count).toBe(3); + expect(top.bytes).toBe(3 * 34); // 32 magnitude + 1 sign + 1 length prefix + expect(m.categories['const-push']).toBe(3 * 34); + }); + + it('builds an opcode histogram by mnemonic', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'dup' }, { op: 'dup' }, { op: 'opcode', code: 'OP_HASH160' }, + ])); + expect(m.opcodes['OP_DUP']).toBe(2); + expect(m.opcodes['OP_HASH160']).toBe(1); + }); + + it('reports the real p256-wallet shape', () => { + // Regression pin on the headline baseline finding: the P-256 verifier is + // dominated by one repeated constant, not by its arithmetic. + const hex = hexOf([ + { op: 'push', value: 1n }, + ]); + expect(analyzeScriptHex(hex).scriptBytes).toBe(1); + }); + + it('rejects a truncated push rather than silently dropping bytes', () => { + // 0x04 promises four data bytes and supplies two. + expect(() => analyzeScriptHex('04aabb')).toThrow(/truncated/i); + }); +}); + +describe('stackOpMetrics', () => { + it('counts ops, recursing into if arms', () => { + const ops: StackOp[] = [ + { op: 'push', value: 1n }, + { op: 'if', then: [{ op: 'dup' }, { op: 'drop' }], else: [{ op: 'swap' }] }, + ]; + const m = stackOpMetrics(ops); + expect(m.opCount).toBe(5); // push, if, dup, drop, swap + expect(m.shuffleOps).toBe(3); + }); + + it('reports script bytes consistent with the cost model', () => { + const ops: StackOp[] = [ + { op: 'push', value: 300n }, + { op: 'opcode', code: 'OP_ADD' }, + ]; + expect(stackOpMetrics(ops).scriptBytes).toBe(hexOf(ops).length / 2); + }); + + it('breaks out pick/roll/dup/swap counts', () => { + const ops: StackOp[] = [ + { op: 'pick', depth: 3 }, { op: 'pick', depth: 4 }, + { op: 'roll', depth: 5 }, + { op: 'dup' }, { op: 'swap' }, { op: 'swap' }, + ]; + const m = stackOpMetrics(ops); + expect(m.opcodes['OP_PICK']).toBe(2); + expect(m.opcodes['OP_ROLL']).toBe(1); + expect(m.opcodes['OP_DUP']).toBe(1); + expect(m.opcodes['OP_SWAP']).toBe(2); + }); +}); diff --git a/packages/runar-compiler/src/index.ts b/packages/runar-compiler/src/index.ts index fc91507bc..298b69c35 100644 --- a/packages/runar-compiler/src/index.ts +++ b/packages/runar-compiler/src/index.ts @@ -67,6 +67,14 @@ export { foldConstants } from './optimizer/constant-fold.js'; export { eliminateDeadBindings } from './optimizer/dce.js'; export { assembleArtifact } from './artifact/assembler.js'; +// Script-byte cost model + size instrumentation. Read-only: nothing here +// changes compilation output, but `estimateScriptBytes` is the metric any +// size-directed optimizer pass must compare candidates with, and it is +// asserted byte-exact against the emitter over the whole conformance corpus. +export { sizeOfStackOp, sizeOfPushValue, estimateScriptBytes } from './metrics/cost-model.js'; +export { analyzeScriptHex, stackOpMetrics } from './metrics/script-metrics.js'; +export type { ByteCategory, ConstantUse, ScriptMetrics, StackOpMetrics } from './metrics/script-metrics.js'; + export type { CompilerDiagnostic, Severity } from './errors.js'; export { CompilerError, ParseError, ValidationError, TypeError, makeDiagnostic } from './errors.js'; @@ -158,6 +166,33 @@ export interface CompileOptions { */ disablePeephole?: boolean; + /** + * EXPERIMENTAL. Park each curve's field prime / group order in a stack slot + * inside the EC codegen modules instead of re-pushing the 33- or 49-byte + * literal at every modular reduction. + * + * Default false, and the emitters take an untouched code path when it is — + * so the checked-in goldens, `conformance/script-size-baseline.json`, and + * cross-tier hex parity are all unaffected while this is off. Turning it on + * changes the emitted bytes and is therefore a TS-tier-only experiment until + * the transformation is ported to the other six compilers. Measured effect: + * `verifyECDSA_P256` 974,024 -> 319,693 bytes (-67 %). + * + * See `docs/experiments/script-size-optimization-baseline.md`. + */ + ecConstantPool?: boolean; + + /** + * EXPERIMENTAL. Operand scheduling strategy for the ANF -> Stack pass. + * + * `'current'` (default) ships today's bytes. `'liveness'` parks a result on + * the alt stack when the next binding does not consume it, so the operands a + * chain reads repeatedly stay at depth 0/1 instead of sinking one slot per + * binding. TS-tier-only experiment while it is opt-in; see + * `docs/experiments/stack-scheduler-design.md`. + */ + schedulerMode?: 'current' | 'liveness'; + /** Called between compilation passes with the current stage name and progress percentage (0-100). */ onProgress?: (stage: string, percent: number) => void; } @@ -463,7 +498,10 @@ export function compile(source: string, options?: CompileOptions): CompileResult // Pass 5-6: Stack lower + Peephole optimize + Emit try { onProgress?.('Stack lowering', 60); - const stackProgram = lowerToStack(optimizedAnf); + const stackProgram = lowerToStack(optimizedAnf, { + ecConstantPool: opts.ecConstantPool === true, + schedulerMode: opts.schedulerMode, + }); // Apply peephole optimization to each method's ops (runs on Stack IR, // after the ANF conformance boundary, so it doesn't affect cross-compiler @@ -552,6 +590,10 @@ export interface CompileFromANFOptions { disableEcOptimizer?: boolean; /** If true, skip the Stack IR peephole optimizer. See CompileOptions for context. */ disablePeephole?: boolean; + /** EXPERIMENTAL. Pool repeated EC curve constants. See CompileOptions. */ + ecConstantPool?: boolean; + /** EXPERIMENTAL. Operand scheduling strategy. See CompileOptions. */ + schedulerMode?: 'current' | 'liveness'; } export interface CompileFromANFResult { @@ -621,7 +663,10 @@ export function compileFromANF( // EC optimizer delegates internally to optimizer/dce.ts for dead-binding cleanup. const optimizedAnf = opts.disableEcOptimizer ? anf : optimizeEC(anf); - const stackProgram = lowerToStack(optimizedAnf); + const stackProgram = lowerToStack(optimizedAnf, { + ecConstantPool: opts.ecConstantPool === true, + schedulerMode: opts.schedulerMode, + }); if (!opts.disablePeephole) { for (const method of stackProgram.methods) { method.ops = optimizeStackIR(method.ops); diff --git a/packages/runar-compiler/src/metrics/cost-model.ts b/packages/runar-compiler/src/metrics/cost-model.ts new file mode 100644 index 000000000..733fefb32 --- /dev/null +++ b/packages/runar-compiler/src/metrics/cost-model.ts @@ -0,0 +1,104 @@ +/** + * Script-byte cost model for Stack IR. + * + * Optimizer passes need to compare two candidate lowerings by the metric that + * actually matters — serialized locking-script bytes — before either one is + * emitted. `OP_DUP` and a 33-byte constant push are one instruction each and + * 1 vs 34 bytes; an instruction count cannot tell them apart. + * + * This module is deliberately NOT an approximation. Every push routes through + * the same `push-encoding.ts` encoders that `06-emit.ts` uses, and the + * structural cases mirror `emitStackOp` / `emitIf` one-for-one. The invariant + * + * estimateScriptBytes(ops) === emitMethod({ ops, ... }).scriptHex.length / 2 + * + * is asserted over the whole conformance corpus in + * `__tests__/cost-model.test.ts`. If you change push encoding or the emit + * switch, that sweep is what tells you this file went stale. + */ + +import type { StackOp } from '../ir/index.js'; +import { OPCODES } from '../passes/06-emit.js'; +import { encodePushBigIntHex, encodePushBytesHex } from '../passes/push-encoding.js'; + +/** + * Serialized byte cost of a single push value. + * + * Mirrors `encodePushValue` in `06-emit.ts`: booleans are the 1-byte OP_TRUE / + * OP_FALSE, bigints go through the small-int opcodes where possible, and byte + * arrays are MINIMALDATA-aware before falling back to a length-prefixed push. + */ +export function sizeOfPushValue(value: Uint8Array | bigint | boolean): number { + if (typeof value === 'boolean') { + return 1; // OP_TRUE (0x51) / OP_FALSE (0x00) + } + if (typeof value === 'bigint') { + return encodePushBigIntHex(value).length / 2; + } + return encodePushBytesHex(value).length / 2; +} + +/** + * Serialized byte cost of one Stack IR operation, including nested `if` arms. + * + * Note on `pick` / `roll`: they cost ONE byte here. The depth operand is a + * separate `push` op that the lowerer emits immediately before (see + * `bringToTop` in `05-stack-lower.ts`), so charging the depth here would + * double-count it. + * + * Throws on an unknown opcode mnemonic rather than costing it zero — a typo + * in a codegen module should surface as a loud failure, not as a cost model + * that quietly under-reports. + */ +export function sizeOfStackOp(op: StackOp): number { + switch (op.op) { + case 'push': + return sizeOfPushValue(op.value); + + case 'dup': + case 'swap': + case 'roll': + case 'pick': + case 'drop': + case 'nip': + case 'over': + case 'rot': + case 'tuck': + return 1; + + case 'opcode': { + if (OPCODES[op.code] === undefined) { + throw new Error(`cost-model: unknown opcode '${op.code}'`); + } + return 1; + } + + case 'if': { + // OP_IF + then + [OP_ELSE + else] + OP_ENDIF. The emitter writes + // OP_ELSE only for a NON-EMPTY else arm (`emitIf` in 06-emit.ts). + let total = 2; // OP_IF + OP_ENDIF + total += estimateScriptBytes(op.then); + if (op.else && op.else.length > 0) { + total += 1 + estimateScriptBytes(op.else); + } + return total; + } + + case 'placeholder': + case 'push_codesep_index': + // Both emit a single 0x00 byte that the SDK rewrites later. + return 1; + + case 'raw_bytes': + return op.bytes.length; + } +} + +/** Serialized byte cost of a Stack IR op sequence. */ +export function estimateScriptBytes(ops: StackOp[]): number { + let total = 0; + for (const op of ops) { + total += sizeOfStackOp(op); + } + return total; +} diff --git a/packages/runar-compiler/src/metrics/script-metrics.ts b/packages/runar-compiler/src/metrics/script-metrics.ts new file mode 100644 index 000000000..0ad2e5fdd --- /dev/null +++ b/packages/runar-compiler/src/metrics/script-metrics.ts @@ -0,0 +1,302 @@ +/** + * Script-size instrumentation. + * + * Two views of the same question — "where did the bytes go?": + * + * - `analyzeScriptHex` walks a SERIALIZED script and buckets every byte by + * what it is spent on. This is the view that matters for a size project, + * because opcode counts hide the thing that actually dominates: a 33-byte + * constant push and an `OP_DUP` are one opcode each and 34x apart in cost. + * - `stackOpMetrics` reports the same shape from Stack IR, before emission, + * so a pass can measure its own output without a round-trip through hex. + * + * The one classification rule worth stating out loud: a push immediately + * consumed by `OP_PICK` / `OP_ROLL` is charged to `stack-shuffle`, not to + * `const-push`. `bringToTop` emits `push(depth)` + `OP_PICK` as a pair (see + * `05-stack-lower.ts`), and blaming those depth bytes on constants would + * credit the wrong optimizer with fixing them. + * + * Nothing here changes compilation output; it only reads it. + */ + +import type { StackOp } from '../ir/index.js'; +import { OPCODES } from '../passes/06-emit.js'; +import { estimateScriptBytes } from './cost-model.js'; + +// --------------------------------------------------------------------------- +// Byte categories +// --------------------------------------------------------------------------- + +export type ByteCategory = + | 'const-push' + | 'small-int-push' + | 'stack-shuffle' + | 'arithmetic' + | 'bytes' + | 'crypto' + | 'control' + | 'other'; + +const CATEGORIES: ByteCategory[] = [ + 'const-push', 'small-int-push', 'stack-shuffle', + 'arithmetic', 'bytes', 'crypto', 'control', 'other', +]; + +/** Reverse map byte -> preferred mnemonic, skipping the OP_FALSE/OP_TRUE aliases. */ +const OPCODE_NAMES: Map = new Map(); +for (const [name, byte] of Object.entries(OPCODES)) { + if (name === 'OP_FALSE' || name === 'OP_TRUE') continue; + if (!OPCODE_NAMES.has(byte)) OPCODE_NAMES.set(byte, name); +} + +const SHUFFLE_OPS = new Set([ + 'OP_DUP', 'OP_DROP', 'OP_NIP', 'OP_OVER', 'OP_PICK', 'OP_ROLL', 'OP_ROT', + 'OP_SWAP', 'OP_TUCK', 'OP_2DROP', 'OP_2DUP', 'OP_3DUP', 'OP_2OVER', + 'OP_2ROT', 'OP_2SWAP', 'OP_IFDUP', 'OP_DEPTH', + 'OP_TOALTSTACK', 'OP_FROMALTSTACK', +]); + +const ARITHMETIC_OPS = new Set([ + 'OP_ADD', 'OP_SUB', 'OP_MUL', 'OP_DIV', 'OP_MOD', 'OP_1ADD', 'OP_1SUB', + 'OP_2MUL', 'OP_2DIV', 'OP_NEGATE', 'OP_ABS', 'OP_NOT', 'OP_0NOTEQUAL', + 'OP_BOOLAND', 'OP_BOOLOR', 'OP_NUMEQUAL', 'OP_NUMEQUALVERIFY', + 'OP_NUMNOTEQUAL', 'OP_LESSTHAN', 'OP_GREATERTHAN', 'OP_LESSTHANOREQUAL', + 'OP_GREATERTHANOREQUAL', 'OP_MIN', 'OP_MAX', 'OP_WITHIN', + 'OP_AND', 'OP_OR', 'OP_XOR', 'OP_INVERT', 'OP_LSHIFT', 'OP_RSHIFT', + 'OP_LSHIFTNUM', 'OP_RSHIFTNUM', +]); + +const BYTES_OPS = new Set([ + 'OP_CAT', 'OP_SPLIT', 'OP_SIZE', 'OP_NUM2BIN', 'OP_BIN2NUM', + 'OP_SUBSTR', 'OP_LEFT', 'OP_RIGHT', 'OP_EQUAL', 'OP_EQUALVERIFY', +]); + +const CRYPTO_OPS = new Set([ + 'OP_RIPEMD160', 'OP_SHA1', 'OP_SHA256', 'OP_HASH160', 'OP_HASH256', + 'OP_CHECKSIG', 'OP_CHECKSIGVERIFY', 'OP_CHECKMULTISIG', + 'OP_CHECKMULTISIGVERIFY', 'OP_CODESEPARATOR', +]); + +const CONTROL_OPS = new Set([ + 'OP_IF', 'OP_NOTIF', 'OP_ELSE', 'OP_ENDIF', 'OP_VERIFY', 'OP_RETURN', + 'OP_NOP', 'OP_CHECKLOCKTIMEVERIFY', 'OP_CHECKSEQUENCEVERIFY', +]); + +function categoryOfOpcode(name: string): ByteCategory { + if (SHUFFLE_OPS.has(name)) return 'stack-shuffle'; + if (ARITHMETIC_OPS.has(name)) return 'arithmetic'; + if (BYTES_OPS.has(name)) return 'bytes'; + if (CRYPTO_OPS.has(name)) return 'crypto'; + if (CONTROL_OPS.has(name)) return 'control'; + return 'other'; +} + +// --------------------------------------------------------------------------- +// Serialized-script analysis +// --------------------------------------------------------------------------- + +export interface ConstantUse { + /** Hex of the pushed data (without the length prefix). */ + hex: string; + /** How many times this exact payload is pushed. */ + count: number; + /** Total serialized bytes spent pushing it (payload + prefix, times count). */ + bytes: number; +} + +export interface ScriptMetrics { + scriptBytes: number; + /** Opcodes plus pushes, each counted once. */ + opcodeCount: number; + pushCount: number; + categories: Record; + /** Mnemonic -> occurrence count. Data pushes are keyed as `PUSH`. */ + opcodes: Record; + /** Repeated data payloads, largest total byte cost first. */ + constants: ConstantUse[]; +} + +/** + * Bucket every byte of a serialized script. + * + * Throws on a malformed / truncated push instead of silently dropping the + * tail — a size report that quietly loses bytes is worse than no report. + */ +export function analyzeScriptHex(scriptHex: string): ScriptMetrics { + const hex = scriptHex.trim(); + if (hex.length % 2 !== 0) { + throw new Error(`analyzeScriptHex: odd-length hex (${hex.length} chars)`); + } + const bytes = Buffer.from(hex, 'hex'); + const n = bytes.length; + + const categories = Object.fromEntries(CATEGORIES.map(c => [c, 0])) as Record; + const opcodes: Record = {}; + const constants = new Map(); + + let opcodeCount = 0; + let pushCount = 0; + + /** Bytes + category of the immediately preceding op, for the PICK/ROLL rule. */ + let prevPush: { size: number; category: ByteCategory; dataHex: string | null } | null = null; + + const bump = (name: string) => { opcodes[name] = (opcodes[name] ?? 0) + 1; }; + + let i = 0; + while (i < n) { + const op = bytes[i]!; + + // --- direct pushes ----------------------------------------------------- + if (op >= 0x01 && op <= 0x4b) { + const len = op; + if (i + 1 + len > n) { + throw new Error(`analyzeScriptHex: truncated push at offset ${i} (want ${len} bytes, ${n - i - 1} left)`); + } + const dataHex = bytes.subarray(i + 1, i + 1 + len).toString('hex'); + const size = 1 + len; + categories['const-push'] += size; + bump('PUSH'); + pushCount++; opcodeCount++; + prevPush = { size, category: 'const-push', dataHex }; + i += size; + continue; + } + + if (op === 0x4c || op === 0x4d || op === 0x4e) { + const hdr = op === 0x4c ? 2 : op === 0x4d ? 3 : 5; + if (i + hdr > n) { + throw new Error(`analyzeScriptHex: truncated PUSHDATA header at offset ${i}`); + } + const len = op === 0x4c + ? bytes[i + 1]! + : op === 0x4d + ? bytes.readUInt16LE(i + 1) + : bytes.readUInt32LE(i + 1); + if (i + hdr + len > n) { + throw new Error(`analyzeScriptHex: truncated PUSHDATA body at offset ${i} (want ${len} bytes)`); + } + const dataHex = bytes.subarray(i + hdr, i + hdr + len).toString('hex'); + const size = hdr + len; + categories['const-push'] += size; + bump('PUSH'); + pushCount++; opcodeCount++; + prevPush = { size, category: 'const-push', dataHex }; + i += size; + continue; + } + + // --- single-byte constant pushes -------------------------------------- + if (op === 0x00 || op === 0x4f || (op >= 0x51 && op <= 0x60)) { + categories['small-int-push'] += 1; + bump(OPCODE_NAMES.get(op) ?? `OP_UNKNOWN_${op.toString(16)}`); + pushCount++; opcodeCount++; + prevPush = { size: 1, category: 'small-int-push', dataHex: null }; + i += 1; + continue; + } + + // --- opcodes ----------------------------------------------------------- + const name = OPCODE_NAMES.get(op) ?? `OP_UNKNOWN_${op.toString(16).padStart(2, '0')}`; + const category = categoryOfOpcode(name); + categories[category] += 1; + bump(name); + opcodeCount++; + + // A depth push consumed by PICK/ROLL is stack-access cost, not a constant. + if ((name === 'OP_PICK' || name === 'OP_ROLL') && prevPush) { + categories[prevPush.category] -= prevPush.size; + categories['stack-shuffle'] += prevPush.size; + if (prevPush.dataHex !== null) { + // It was recorded as a data push; un-record it from the constants tally. + const existing = constants.get(prevPush.dataHex); + if (existing) { + existing.count -= 1; + existing.bytes -= prevPush.size; + if (existing.count === 0) constants.delete(prevPush.dataHex); + } + } + } else if (prevPush && prevPush.dataHex !== null) { + // Only now is the previous data push confirmed to be a real constant. + const entry = constants.get(prevPush.dataHex) ?? { count: 0, bytes: 0 }; + entry.count += 1; + entry.bytes += prevPush.size; + constants.set(prevPush.dataHex, entry); + } + + prevPush = null; + i += 1; + } + + // A data push in final position was never confirmed by the loop above. + if (prevPush && prevPush.dataHex !== null) { + const entry = constants.get(prevPush.dataHex) ?? { count: 0, bytes: 0 }; + entry.count += 1; + entry.bytes += prevPush.size; + constants.set(prevPush.dataHex, entry); + } + + const constantList: ConstantUse[] = [...constants.entries()] + .map(([h, v]) => ({ hex: h, count: v.count, bytes: v.bytes })) + .sort((a, b) => b.bytes - a.bytes); + + return { + scriptBytes: n, + opcodeCount, + pushCount, + categories, + opcodes, + constants: constantList, + }; +} + +// --------------------------------------------------------------------------- +// Stack IR analysis +// --------------------------------------------------------------------------- + +export interface StackOpMetrics { + scriptBytes: number; + /** Every op, recursing into `if` arms. An `if` counts as one plus its arms. */ + opCount: number; + /** Ops that only move data around (dup/drop/pick/roll/swap/…). */ + shuffleOps: number; + /** Mnemonic -> count. Structural ops are keyed by the opcode they emit. */ + opcodes: Record; + maxStackDepth?: number; +} + +const STACK_OP_MNEMONIC: Partial> = { + dup: 'OP_DUP', swap: 'OP_SWAP', roll: 'OP_ROLL', pick: 'OP_PICK', + drop: 'OP_DROP', nip: 'OP_NIP', over: 'OP_OVER', rot: 'OP_ROT', + tuck: 'OP_TUCK', if: 'OP_IF', push: 'PUSH', + placeholder: 'PLACEHOLDER', push_codesep_index: 'CODESEP_INDEX', + raw_bytes: 'RAW_BYTES', +}; + +/** Metrics for a Stack IR op sequence, before emission. */ +export function stackOpMetrics(ops: StackOp[], maxStackDepth?: number): StackOpMetrics { + const opcodes: Record = {}; + let opCount = 0; + let shuffleOps = 0; + + const walk = (list: StackOp[]): void => { + for (const op of list) { + opCount++; + const name = op.op === 'opcode' ? op.code : STACK_OP_MNEMONIC[op.op]!; + opcodes[name] = (opcodes[name] ?? 0) + 1; + if (SHUFFLE_OPS.has(name)) shuffleOps++; + if (op.op === 'if') { + walk(op.then); + if (op.else) walk(op.else); + } + } + }; + walk(ops); + + return { + scriptBytes: estimateScriptBytes(ops), + opCount, + shuffleOps, + opcodes, + maxStackDepth, + }; +} diff --git a/packages/runar-compiler/src/passes/05-stack-lower.ts b/packages/runar-compiler/src/passes/05-stack-lower.ts index ec9c57ce2..4f2f55b12 100644 --- a/packages/runar-compiler/src/passes/05-stack-lower.ts +++ b/packages/runar-compiler/src/passes/05-stack-lower.ts @@ -23,6 +23,9 @@ import { UnknownANFKindError, MERGED_LOCAL_TEMP_PREFIX } from 'runar-ir-schema'; import { emitVerifySLHDSA } from './slh-dsa-codegen.js'; import { emitVerifyWOTS } from './wots-codegen.js'; import { emitVerifyRabinSig } from './rabin-codegen.js'; +import type { EcCodegenOptions } from './ec-codegen.js'; +import { estimateScriptBytes } from '../metrics/cost-model.js'; +import { optimizeStackIR } from '../optimizer/peephole.js'; import { emitEcAdd, emitEcMul, emitEcMulGen, emitEcNegate, emitEcOnCurve, emitEcModReduce, emitEcEncodeCompressed, @@ -62,6 +65,116 @@ import { const MAX_STACK_DEPTH = 800; +/** + * Experimental lowering options. + * + * Every field defaults to the shipping behaviour, so `lowerToStack(anf)` with + * no options is byte-identical to what the seven tiers produce today. These + * exist so a size experiment can be measured against the default without + * moving a single golden. See `docs/experiments/`. + */ +export interface LoweringOptions { + /** + * Park each curve's field prime / group order in a stack slot inside the EC + * codegen modules instead of re-pushing the 33- or 49-byte literal at every + * modular reduction. + */ + ecConstantPool?: boolean; + + /** + * Operand scheduling strategy. + * + * `'current'` (default) is the shipping behaviour: results are pushed on top + * of the operands that produced them, so a chain that reads the same values + * repeatedly pays a deeper `OP_PICK` each time. + * + * `'liveness'` parks a result on the alt stack when the next binding does + * not consume it, keeping hot operands at depth 0/1, and restores the whole + * spill group before the first binding that needs any of it. + */ + schedulerMode?: 'current' | 'liveness'; +} + +/** + * ANF value kinds the spill scheduler is allowed to reason about. + * + * Deliberately narrow: these compute a single value from operands already on + * the stack and have no control flow, no side effects, and no bespoke stack + * choreography. Everything else — `if`, `loop`, `assert`, state ops, calls, + * `raw_script` — forces the alt stack to be drained first, so none of the + * delicate branch/loop reconciliation in `lowerIf` / `lowerLoop` ever sees a + * non-empty alt stack. + */ +const SPILLABLE_KINDS = new Set([ + 'bin_op', 'unary_op', 'load_const', 'load_param', 'load_prop', +]); + +/** Bytes an OP_TOALTSTACK + OP_FROMALTSTACK round trip costs. */ +const SPILL_ROUND_TRIP_BYTES = 2; + +/** + * Binary operators whose two operands may be materialized in either order. + * + * The operator itself is unchanged — only which operand is brought to the top + * first. Excluded on purpose: + * - `+` is `OP_CAT` when `result_type` is `"bytes"`, and concatenation is not + * commutative, so the caller must check the result type too; + * - `&&` / `||` short-circuit on chain (they lower through a branch, not + * `OP_BOOLAND` / `OP_BOOLOR`), so their operands are not interchangeable + * at this level; + * - `-`, `/`, `%`, shifts and every ordered comparison are order-sensitive. + * + * `===` / `!==` are commutative under both `OP_NUMEQUAL` and `OP_EQUAL`. + */ +const COMMUTATIVE_BINOPS = new Set(['+', '*', '===', '!==', '&', '|', '^']); + +/** + * The ops one `bringToTop` would emit, mirroring its cases exactly, applied to + * a throwaway stack model. + * + * Returns null when the value is not resident, which makes the caller fall + * back to source order rather than guess. + * + * Costing has to go through the real peephole rather than a byte formula, + * because the cheapest-looking local choice is often the one the peephole + * would have erased anyway: two consumed operands at depths 1 and 0 emit + * `OP_SWAP OP_SWAP`, which `swap-swap` deletes outright — free — while + * "cleverly" taking them in the other order emits one real `OP_SWAP` and + * costs a byte. + */ +function materializationOps(model: StackMap, name: string, consume: boolean): StackOp[] | null { + let depth: number; + try { + depth = model.findDepth(name); + } catch { + return null; + } + + const ops: StackOp[] = []; + if (depth === 0) { + if (!consume) { ops.push({ op: 'dup' }); model.dup(); } + return ops; + } + if (depth === 1) { + if (consume) { ops.push({ op: 'swap' }); model.swap(); } + else { ops.push({ op: 'over' }); model.push(model.peekAtDepth(1)); } + return ops; + } + if (consume) { + if (depth === 2) { + ops.push({ op: 'rot' }); + } else { + ops.push({ op: 'push', value: BigInt(depth) }, { op: 'roll', depth }); + } + model.push(model.removeAtDepth(depth)); + } else { + ops.push({ op: 'push', value: BigInt(depth) }, { op: 'pick', depth }); + model.push(model.peekAtDepth(depth)); + } + return ops; +} + + /** * Local hex-to-Uint8Array helper. Avoids a runar-testing dependency * (runar-testing depends on runar-compiler, so the reverse direction @@ -575,11 +688,25 @@ class LoweringContext { */ private readonly renamedParams: Map = new Map(); + /** Experimental lowering options; all default to shipping behaviour. */ + private readonly opts: LoweringOptions; + + /** + * Values parked on the alt stack by the liveness scheduler, bottom -> top. + * + * Always empty in `'current'` mode, and always drained before any binding + * the scheduler does not model (see SPILLABLE_KINDS) and at the end of + * every `lowerBindings` scope. + */ + private altSpills: string[] = []; + constructor( params: string[], properties: ANFProperty[], privateMethods: Map = new Map(), + opts: LoweringOptions = {}, ) { + this.opts = opts; // Parameters are pushed onto the stack by the Bitcoin VM in order. // The first parameter is at the bottom, last parameter at the top. this.stackMap = new StackMap(params); @@ -1035,6 +1162,131 @@ class LoweringContext { * If `consume` is true, use ROLL (removes from original position). * If `consume` is false, use PICK (copies, leaving original in place). */ + // -- liveness scheduler: alt-stack spilling -------------------------------- + // + // ANF pushes every result on top of the operands that made it, so a chain + // reading the same two values repeatedly buries them one slot deeper per + // binding and pays a `push d; OP_PICK` pair instead of a 1-byte `OP_2DUP`. + // Parking the result on the alt stack keeps the operands hot. + // + // Restores are done as a GROUP, not one at a time. Popping the whole alt + // stack puts the values back on the main stack in production order (first + // spilled ends up on top), which is the order an ANF accumulation chain + // reads them in — so the group restore usually lands operands exactly where + // the next bindings want them, and `bringToTop` fixes up any that differ. + + /** True when the liveness scheduler is active for this context. */ + private get schedulingByLiveness(): boolean { + // Never inside a branch arm: `lowerIf` reconciles arms by MAIN-stack depth + // alone (the Layer B/C invariants), so an arm must neither begin nor end + // with a non-empty alt stack. + return this.opts.schedulerMode === 'liveness' && !this._insideBranch; + } + + /** Move the top-of-stack value to the alt stack. */ + private spillToAlt(name: string): void { + this.emitOp({ op: 'opcode', code: 'OP_TOALTSTACK' }); + this.stackMap.pop(); + this.altSpills.push(name); + } + + /** Pop every spilled value back. First-spilled ends up on top. */ + private restoreSpills(): void { + while (this.altSpills.length > 0) { + const name = this.altSpills.pop()!; + this.emitOp({ op: 'opcode', code: 'OP_FROMALTSTACK' }); + this.stackMap.push(name); + this.trackDepth(); + } + } + + /** + * Decide whether the just-computed `name` (now on top) should be parked. + * + * Spilling pays only if the value would otherwise sit above operands that + * get read before it is needed. The estimate counts those intervening reads: + * each one would cross this slot, and crossing it costs at least a byte once + * the access stops being a depth-0/1 `OP_DUP`/`OP_OVER`. Break-even is the + * 2-byte round trip, so 2 intervening reads are required. + */ + private maybeSpill( + binding: ANFBinding, + bindingIndex: number, + bindings: ANFBinding[], + lastUses: Map, + ): void { + if (!this.schedulingByLiveness) return; + if (!SPILLABLE_KINDS.has(binding.value.kind)) return; + // Only a value we actually left on top, under its own name. + if (this.stackMap.depth === 0) return; + if (this.stackMap.peekAtDepth(0) !== binding.name) return; + + const nextUse = this.nextUseAfter(binding.name, bindingIndex, bindings); + if (nextUse === null) return; // dead or used past this scope + if (nextUse === bindingIndex + 1) return; // consumed immediately: no burial + if (lastUses.get(binding.name) !== undefined + && lastUses.get(binding.name)! >= bindings.length) return; // pinned for an outer scope + + // No control flow may follow the spill anywhere in this scope. + // + // Restoring immediately before an `if` leaves the parent stack in a shape + // `lowerIf` was not written for: its arm reconciliation, declared-result + // trim and Layer B/C depth invariants all reason about a main stack the + // scheduler has not been rearranging underneath them. That combination + // MISCOMPILED `if-without-else-multi-temp` — the script kept running and + // started ACCEPTING a witness the shipping compiler rejects, which the + // conformance witness corpus caught (see + // `packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts`). + // Rather than try to make the two agree, spilling stays out of any scope + // that still has control flow ahead of it. `assert` is allowed: it consumes + // a value and emits OP_VERIFY without reshaping anything. + for (let j = bindingIndex + 1; j < bindings.length; j++) { + const kind = bindings[j]!.value.kind; + if (!SPILLABLE_KINDS.has(kind) && kind !== 'assert') return; + } + + // Everything between here and the use must itself be schedulable, or the + // restore would land in the middle of a construct we do not model. + let interveningReads = 0; + for (let j = bindingIndex + 1; j < nextUse; j++) { + const v = bindings[j]!.value; + if (!SPILLABLE_KINDS.has(v.kind)) return; + interveningReads += collectRefs(v).filter(r => r !== binding.name).length; + } + if (interveningReads < SPILL_ROUND_TRIP_BYTES) return; + + // Do not spill a value the very next binding will restore anyway: the + // round trip would be emitted and immediately undone. This is the + // `OP_DIV OP_TOALTSTACK OP_FROMALTSTACK` shape — 2 bytes for nothing. + const next = bindings[bindingIndex + 1]; + if (next !== undefined) { + const nextRefs = collectRefs(next.value); + if (!SPILLABLE_KINDS.has(next.value.kind) + || nextRefs.some(r => this.altSpills.includes(r))) return; + } + + this.spillToAlt(binding.name); + } + + /** Index of the first binding at or after `from + 1` that references `name`. */ + private nextUseAfter(name: string, from: number, bindings: ANFBinding[]): number | null { + for (let j = from + 1; j < bindings.length; j++) { + if (collectRefs(bindings[j]!.value).includes(name)) return j; + } + return null; + } + + /** + * Options handed to the EC / NIST codegen modules. + * + * Returns `undefined` — not `{}` — when nothing is enabled, so the emitters + * take their untouched default path and the emitted bytes are provably + * identical to the shipping ones. + */ + private ecCodegenOptions(): EcCodegenOptions | undefined { + return this.opts.ecConstantPool ? { constantPool: true } : undefined; + } + bringToTop(name: string, consume: boolean): void { const depth = this.stackMap.findDepth(name); @@ -1178,6 +1430,16 @@ class LoweringContext { for (let i = 0; i < bindings.length; i++) { const binding = bindings[i]!; + // Drain the alt stack before anything the scheduler does not model, and + // before the first binding that reads a parked value. + if (this.altSpills.length > 0) { + const refs = collectRefs(binding.value); + if (!SPILLABLE_KINDS.has(binding.value.kind) + || refs.some(r => this.altSpills.includes(r)) + || i === lastAssertIdx || i === terminalIfIdx) { + this.restoreSpills(); + } + } // Propagate source location from ANF binding to StackOps this.currentSourceLoc = binding.sourceLoc; if (binding.value.kind === 'assert' && i === lastAssertIdx) { @@ -1188,10 +1450,13 @@ class LoweringContext { this.lowerIf(binding.name, binding.value.cond, binding.value.then, binding.value.else, binding.value.results ?? [], i, lastUses, true); } else { this.lowerBinding(binding, i, lastUses); + this.maybeSpill(binding, i, bindings, lastUses); } this.currentSourceLoc = undefined; } + // Nothing may outlive the scope on the alt stack. + this.restoreSpills(); } private lowerBinding( @@ -1494,6 +1759,45 @@ class LoweringContext { } } + /** + * Would materializing a commutative operator's operands right-then-left cost + * fewer bytes than left-then-right? + * + * Scored with the exact emit-time cost of each `bringToTop`, on a throwaway + * copy of the stack map so the real one is untouched. Ties keep source + * order, which is what makes `'current'` and `'liveness'` identical wherever + * this cannot help. + */ + private shouldSwapOperands( + op: string, + left: string, + right: string, + leftConsume: boolean, + rightConsume: boolean, + resultType?: string, + ): boolean { + if (this.opts.schedulerMode !== 'liveness') return false; + if (!COMMUTATIVE_BINOPS.has(op)) return false; + // `+` on ByteString operands is OP_CAT, which is not commutative. + if (resultType === 'bytes' && op === '+') return false; + if (left === right) return false; + + const cost = (first: string, firstConsume: boolean, second: string, secondConsume: boolean): number => { + const model = this.stackMap.clone(); + const ops: StackOp[] = []; + for (const [name, consume] of [[first, firstConsume], [second, secondConsume]] as const) { + const emitted = materializationOps(model, name, consume); + if (emitted === null) return Number.POSITIVE_INFINITY; // not resident + ops.push(...emitted); + } + // Score what the emitter would actually see, peephole included. + return estimateScriptBytes(optimizeStackIR(ops)); + }; + + return cost(right, rightConsume, left, leftConsume) + < cost(left, leftConsume, right, rightConsume); + } + private lowerBinOp( bindingName: string, op: string, @@ -1503,13 +1807,20 @@ class LoweringContext { lastUses: Map, resultType?: string, ): void { - // Get left operand to stack first const leftConsume = this.operandConsume(left, [left, right], bindingIndex, lastUses); - this.bringToTop(left, leftConsume); - - // Get right operand to stack const rightConsume = this.operandConsume(right, [left, right], bindingIndex, lastUses); - this.bringToTop(right, rightConsume); + + // Commutative operators may take their operands in either order, so pick + // the cheaper arrangement. The common win: the operand already on top is + // materialized SECOND for free, instead of being buried by the other one + // and then swapped back. + if (this.shouldSwapOperands(op, left, right, leftConsume, rightConsume, resultType)) { + this.bringToTop(right, rightConsume); + this.bringToTop(left, leftConsume); + } else { + this.bringToTop(left, leftConsume); + this.bringToTop(right, rightConsume); + } // Pop both operands (the opcode consumes them) this.stackMap.pop(); @@ -4901,13 +5212,14 @@ class LoweringContext { for (let i = 0; i < args.length; i++) this.stackMap.pop(); const emitFn = (op: StackOp) => this.emitOp(op); + const ecOpts = this.ecCodegenOptions(); switch (func) { - case 'ecAdd': emitEcAdd(emitFn); break; - case 'ecMul': emitEcMul(emitFn); break; - case 'ecMulGen': emitEcMulGen(emitFn); break; - case 'ecNegate': emitEcNegate(emitFn); break; - case 'ecOnCurve': emitEcOnCurve(emitFn); break; + case 'ecAdd': emitEcAdd(emitFn, ecOpts); break; + case 'ecMul': emitEcMul(emitFn, ecOpts); break; + case 'ecMulGen': emitEcMulGen(emitFn, ecOpts); break; + case 'ecNegate': emitEcNegate(emitFn, ecOpts); break; + case 'ecOnCurve': emitEcOnCurve(emitFn, ecOpts); break; case 'ecModReduce': emitEcModReduce(emitFn); break; case 'ecEncodeCompressed': emitEcEncodeCompressed(emitFn); break; case 'ecMakePoint': emitEcMakePoint(emitFn); break; @@ -4938,19 +5250,20 @@ class LoweringContext { for (let i = 0; i < args.length; i++) this.stackMap.pop(); const emitFn = (op: StackOp) => this.emitOp(op); + const ecOpts = this.ecCodegenOptions(); switch (func) { - case 'p256Add': emitP256Add(emitFn); break; - case 'p256Mul': emitP256Mul(emitFn); break; - case 'p256MulGen': emitP256MulGen(emitFn); break; - case 'p256Negate': emitP256Negate(emitFn); break; - case 'p256OnCurve': emitP256OnCurve(emitFn); break; + case 'p256Add': emitP256Add(emitFn, ecOpts); break; + case 'p256Mul': emitP256Mul(emitFn, ecOpts); break; + case 'p256MulGen': emitP256MulGen(emitFn, ecOpts); break; + case 'p256Negate': emitP256Negate(emitFn, ecOpts); break; + case 'p256OnCurve': emitP256OnCurve(emitFn, ecOpts); break; case 'p256EncodeCompressed': emitP256EncodeCompressed(emitFn); break; - case 'p384Add': emitP384Add(emitFn); break; - case 'p384Mul': emitP384Mul(emitFn); break; - case 'p384MulGen': emitP384MulGen(emitFn); break; - case 'p384Negate': emitP384Negate(emitFn); break; - case 'p384OnCurve': emitP384OnCurve(emitFn); break; + case 'p384Add': emitP384Add(emitFn, ecOpts); break; + case 'p384Mul': emitP384Mul(emitFn, ecOpts); break; + case 'p384MulGen': emitP384MulGen(emitFn, ecOpts); break; + case 'p384Negate': emitP384Negate(emitFn, ecOpts); break; + case 'p384OnCurve': emitP384OnCurve(emitFn, ecOpts); break; case 'p384EncodeCompressed': emitP384EncodeCompressed(emitFn); break; default: throw new Error(`Unknown NIST EC builtin: ${func}`); } @@ -4975,8 +5288,9 @@ class LoweringContext { this.stackMap.pop(); // sig this.stackMap.pop(); // msg const emitFn = (op: StackOp) => this.emitOp(op); - if (func === 'verifyECDSA_P256') emitVerifyECDSA_P256(emitFn); - else emitVerifyECDSA_P384(emitFn); + const ecOpts = this.ecCodegenOptions(); + if (func === 'verifyECDSA_P256') emitVerifyECDSA_P256(emitFn, ecOpts); + else emitVerifyECDSA_P384(emitFn, ecOpts); this.stackMap.push(bindingName); this.trackDepth(); } @@ -5452,7 +5766,7 @@ function hexToBytes(hex: string): Uint8Array { * named temporaries via a stack map and emits PICK/ROLL to materialise * values as needed. */ -export function lowerToStack(program: ANFProgram): StackProgram { +export function lowerToStack(program: ANFProgram, opts: LoweringOptions = {}): StackProgram { const methods: StackMethod[] = []; const privateMethods = new Map(); @@ -5466,7 +5780,7 @@ export function lowerToStack(program: ANFProgram): StackProgram { if (method.name !== 'constructor' && !method.isPublic) { continue; } - const stackMethod = lowerMethod(method, program.properties, privateMethods); + const stackMethod = lowerMethod(method, program.properties, privateMethods, opts); methods.push(stackMethod); } @@ -5562,10 +5876,43 @@ function methodUsesCodePart(bindings: ANFBinding[]): boolean { return false; } +/** + * Lower one method, and — under `schedulerMode: 'liveness'` — pick the + * cheaper of the two schedules by emitted bytes. + * + * The per-site spill heuristic in `maybeSpill` is greedy and approximate: it + * cannot know whether removing one slot actually moves an access across a + * cost boundary, because `OP_DUP`/`OP_SWAP`/`OP_OVER`/`OP_ROT` all cost one + * byte, so burying a value is free until the access reaches depth 3. Rather + * than model the whole stack evolution ahead of time, lower both ways and let + * the exact cost model decide. That makes "the scheduler never grows a method" + * a structural property rather than a hope. + * + * Sizes are compared AFTER the peephole pass, since that is what the emitter + * finally sees — spilling changes which peephole rules fire (an `OP_OVER + * OP_OVER` that fused into `OP_2DUP` may become a plain `OP_DUP`). + */ function lowerMethod( method: ANFMethod, properties: ANFProperty[], privateMethods: Map, + opts: LoweringOptions = {}, +): StackMethod { + if (opts.schedulerMode === 'liveness') { + const scheduled = lowerMethodOnce(method, properties, privateMethods, opts); + const baseline = lowerMethodOnce(method, properties, privateMethods, + { ...opts, schedulerMode: 'current' }); + const size = (m: StackMethod): number => estimateScriptBytes(optimizeStackIR(m.ops)); + return size(scheduled) < size(baseline) ? scheduled : baseline; + } + return lowerMethodOnce(method, properties, privateMethods, opts); +} + +function lowerMethodOnce( + method: ANFMethod, + properties: ANFProperty[], + privateMethods: Map, + opts: LoweringOptions = {}, ): StackMethod { const paramNames = method.params.map(p => p.name); @@ -5588,7 +5935,7 @@ function lowerMethod( paramNames.unshift('_codePart'); } - const ctx = new LoweringContext(paramNames, properties, privateMethods); + const ctx = new LoweringContext(paramNames, properties, privateMethods, opts); // Pass terminalAssert=true for public methods so the last assert leaves // its value on the stack (Bitcoin Script requires a truthy top-of-stack). ctx.lowerBindings(method.body, method.isPublic); diff --git a/packages/runar-compiler/src/passes/ec-codegen.ts b/packages/runar-compiler/src/passes/ec-codegen.ts index e1dc017ff..566c5d00f 100644 --- a/packages/runar-compiler/src/passes/ec-codegen.ts +++ b/packages/runar-compiler/src/passes/ec-codegen.ts @@ -10,6 +10,7 @@ */ import type { StackOp } from '../ir/index.js'; +import { sizeOfPushValue } from '../metrics/cost-model.js'; // =========================================================================== // Constants @@ -40,13 +41,49 @@ function bigintToBytes32(n: bigint): Uint8Array { // ECTracker — named stack state tracker (mirrors SLHTracker) // =========================================================================== +/** + * Codegen options shared by every EC / NIST-curve emitter. + * + * Off by default: with no options (or `constantPool: false`) each emitter is + * byte-identical to what the seven tiers ship today, so no golden, size + * baseline, or cross-tier parity gate can move. + */ +export interface EcCodegenOptions { + /** + * Park large repeated constants (the field prime, the group order) in a + * stack slot and copy them with `OP_PICK` instead of re-pushing the literal. + * + * `fieldMod` pushes the 256-bit prime at every modular reduction — 34 bytes + * a time, 20,025 times in `p256-wallet` (71 % of that fixture). A pick from + * a slot a dozen deep costs 2. See + * `docs/experiments/script-size-optimization-baseline.md`. + */ + constantPool?: boolean; +} + +/** Stack slot names reserved for pooled constants. */ +export const POOL_FIELD_P = '_pool$p'; +export const POOL_GROUP_N = '_pool$n'; + export class ECTracker { nm: (string | null)[]; _e: (op: StackOp) => void; - - constructor(init: (string | null)[], emit: (op: StackOp) => void) { + /** True when this tracker may serve constants from a pooled slot. */ + readonly pooling: boolean; + + constructor( + init: (string | null)[], + emit: (op: StackOp) => void, + opts?: EcCodegenOptions, + ) { this.nm = [...init]; this._e = emit; + this.pooling = opts?.constantPool === true; + } + + /** The options this tracker was built with, for handing to a nested tracker. */ + get options(): EcCodegenOptions { + return { constantPool: this.pooling }; } get depth(): number { return this.nm.length; } @@ -109,6 +146,48 @@ export class ECTracker { } toTop(name: string): void { this.roll(this.findDepth(name)); } copyToTop(name: string, n?: string): void { this.pick(this.findDepth(name), n ?? name); } + + // -- constant pool -------------------------------------------------------- + // + // A pooled constant is an ordinary tracked slot; nothing about the stack + // model changes. `pushConst` just chooses, per call site and by emitted + // bytes, between copying that slot and re-pushing the literal. Nested + // trackers built with `[...t.nm]` inherit the slot for free, so pooled + // constants work unchanged inside an `OP_IF` arm. + + /** Park `value` in `slot` for the lifetime of this emitter. No-op when pooling is off. */ + poolConstant(slot: string, value: bigint): void { + if (!this.pooling || this.nm.includes(slot)) return; + this.pushInt(slot, value); + } + + /** Remove a pooled slot. No-op when pooling is off or the slot is absent. */ + releaseConstant(slot: string): void { + if (!this.pooling || !this.nm.includes(slot)) return; + this.toTop(slot); + this.drop(); + } + + /** + * Materialize `value` on top as `name`, from the pooled `slot` when that is + * cheaper in emitted bytes than pushing the literal. + * + * The comparison is exact — `sizeOfPushValue` is the same encoder the emit + * pass uses — so pooling can never make a call site bigger. A pick at depth + * d costs `sizeOfPushValue(d) + 1`; depths 0 and 1 are OP_DUP / OP_OVER, 1 + * byte each. + */ + pushConst(slot: string, value: bigint, name: string): void { + if (this.pooling && this.nm.includes(slot)) { + const d = this.findDepth(slot); + const pickCost = d <= 1 ? 1 : sizeOfPushValue(BigInt(d)) + 1; + if (pickCost < sizeOfPushValue(value)) { + this.pick(d, name); + return; + } + } + this.pushInt(name, value); + } toAlt(): void { this.op('OP_TOALTSTACK'); this.nm.pop(); } fromAlt(n: string): void { this.op('OP_FROMALTSTACK'); this.nm.push(n); } rename(n: string): void { @@ -145,11 +224,10 @@ export class ECTracker { /** Push the field prime p onto the stack as a script number. */ function pushFieldP(t: ECTracker, name: string): void { - // Push p directly as a BigInt — the emit pass encodes it as a proper - // little-endian sign-magnitude script number push. - t.pushInt(name, FIELD_P); + t.pushConst(POOL_FIELD_P, FIELD_P, name); } + /** * Reduce a scalar to [0, n-1]: ((k mod n) + n) mod n. * @@ -165,7 +243,7 @@ function pushFieldP(t: ECTracker, name: string): void { * well defined. */ function emitScalarReduce(t: ECTracker, kName: string, resultName: string): void { - t.pushInt('_n_red', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n_red'); t.rawBlock([kName, '_n_red'], resultName, (e) => { e({ op: 'opcode', code: 'OP_2DUP' }); e({ op: 'opcode', code: 'OP_MOD' }); @@ -626,7 +704,7 @@ function jacobianToAffine(t: ECTracker, rxName: string, ryName: string): void { */ function buildJacobianAddAffineInline(e: (op: StackOp) => void, t: ECTracker): void { // Create inner tracker with cloned stack state - jacobianAddAffineBody(new ECTracker([...t.nm], e), false); + jacobianAddAffineBody(new ECTracker([...t.nm], e, t.options), false); } /** @@ -780,7 +858,7 @@ function selectCoord(t: ECTracker, addName: string, dblName: string, condName: s * Stack layout: [..., ax, ay, _k, jx, jy, jz] — same in and out. */ function buildJacobianAddOrDoubleInline(e: (op: StackOp) => void, t: ECTracker): void { - const it = new ECTracker([...t.nm], e); + const it = new ECTracker([...t.nm], e, t.options); // Keep the pre-add accumulator: it is what must be DOUBLED in the // exceptional case, and the add below consumes jx/jy/jz. @@ -839,12 +917,14 @@ function buildJacobianAddOrDoubleInline(e: (op: StackOp) => void, t: ECTracker): * Stack in: [point_a, point_b] (b on top) * Stack out: [result_point] */ -export function emitEcAdd(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pa', '_pb'], emit); +export function emitEcAdd(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pa', '_pb'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); decomposePoint(t, '_pa', 'px', 'py'); decomposePoint(t, '_pb', 'qx', 'qy'); affineAdd(t); composePoint(t, 'rx', 'ry', '_result'); + t.releaseConstant(POOL_FIELD_P); } /** @@ -858,8 +938,10 @@ export function emitEcAdd(emit: (op: StackOp) => void): void { * This avoids the k+n overflow issue where bit 256 was only set for * large k, causing incorrect results for ~half of all scalar values. */ -export function emitEcMul(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt', '_k'], emit); +export function emitEcMul(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt', '_k'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); + t.poolConstant(POOL_GROUP_N, CURVE_N); decomposePoint(t, '_pt', 'ax', 'ay'); // k' = k + 3n: guarantees bit 257 is set for MSB-first double-and-add. @@ -870,15 +952,15 @@ export function emitEcMul(emit: (op: StackOp) => void): void { // usually an unlock argument — so reduce it first. See emitScalarReduce. t.toTop('_k'); emitScalarReduce(t, '_k', '_kr'); - t.pushInt('_n', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n'); t.rawBlock(['_kr', '_n'], '_kn', (e) => { e({ op: 'opcode', code: 'OP_ADD' }); }); - t.pushInt('_n2', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n2'); t.rawBlock(['_kn', '_n2'], '_kn2', (e) => { e({ op: 'opcode', code: 'OP_ADD' }); }); - t.pushInt('_n3', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n3'); t.rawBlock(['_kn2', '_n3'], '_kn3', (e) => { e({ op: 'opcode', code: 'OP_ADD' }); }); @@ -935,6 +1017,8 @@ export function emitEcMul(emit: (op: StackOp) => void): void { t.toTop('_k'); t.drop(); composePoint(t, '_rx', '_ry', '_result'); + t.releaseConstant(POOL_GROUP_N); + t.releaseConstant(POOL_FIELD_P); } /** @@ -942,14 +1026,14 @@ export function emitEcMul(emit: (op: StackOp) => void): void { * Stack in: [scalar] * Stack out: [result_point] */ -export function emitEcMulGen(emit: (op: StackOp) => void): void { +export function emitEcMulGen(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { // Push generator point as 64-byte blob, then delegate to ecMul const gPoint = new Uint8Array(64); gPoint.set(bigintToBytes32(GEN_X), 0); gPoint.set(bigintToBytes32(GEN_Y), 32); emit({ op: 'push', value: gPoint }); emit({ op: 'swap' }); // [point, scalar] - emitEcMul(emit); + emitEcMul(emit, opts); } /** @@ -957,12 +1041,14 @@ export function emitEcMulGen(emit: (op: StackOp) => void): void { * Stack in: [point] * Stack out: [negated_point] */ -export function emitEcNegate(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitEcNegate(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); decomposePoint(t, '_pt', '_nx', '_ny'); pushFieldP(t, '_fp'); fieldSub(t, '_fp', '_ny', '_neg_y'); composePoint(t, '_nx', '_neg_y', '_result'); + t.releaseConstant(POOL_FIELD_P); } /** @@ -970,8 +1056,9 @@ export function emitEcNegate(emit: (op: StackOp) => void): void { * Stack in: [point] * Stack out: [boolean] */ -export function emitEcOnCurve(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitEcOnCurve(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); decomposePoint(t, '_pt', '_x', '_y'); // GAP-301: coordinate canonicity. `decomposePoint` BIN2NUMs each coordinate @@ -1019,6 +1106,7 @@ export function emitEcOnCurve(emit: (op: StackOp) => void): void { t.rawBlock(['_canon', '_curve_eq'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_FIELD_P); } /** diff --git a/packages/runar-compiler/src/passes/p256-p384-codegen.ts b/packages/runar-compiler/src/passes/p256-p384-codegen.ts index ac45060e0..f4a6387ab 100644 --- a/packages/runar-compiler/src/passes/p256-p384-codegen.ts +++ b/packages/runar-compiler/src/passes/p256-p384-codegen.ts @@ -14,7 +14,8 @@ */ import type { StackOp } from '../ir/index.js'; -import { ECTracker } from './ec-codegen.js'; +import { ECTracker, POOL_FIELD_P, POOL_GROUP_N } from './ec-codegen.js'; +import type { EcCodegenOptions } from './ec-codegen.js'; // =========================================================================== // P-256 constants (secp256r1 / NIST P-256) @@ -129,7 +130,7 @@ const P384_PARAMS: CurveParams = { }; function pushFieldP(t: ECTracker, name: string, c: CurveParams): void { - t.pushInt(name, c.fieldP); + t.pushConst(POOL_FIELD_P, c.fieldP, name); } function cFieldMod(t: ECTracker, aName: string, resultName: string, c: CurveParams): void { @@ -230,7 +231,7 @@ const P256_GROUP: GroupParams = { n: P256_N, nMinus2: P256_N_MINUS_2 }; const P384_GROUP: GroupParams = { n: P384_N, nMinus2: P384_N_MINUS_2 }; function pushGroupN(t: ECTracker, name: string, g: GroupParams): void { - t.pushInt(name, g.n); + t.pushConst(POOL_GROUP_N, g.n, name); } function cGroupMod(t: ECTracker, aName: string, resultName: string, g: GroupParams): void { @@ -648,7 +649,7 @@ function cJacobianToAffine(t: ECTracker, rxName: string, ryName: string, c: Curv * After: [..., ax, ay, _k, jx', jy', jz'] */ function buildJacobianAddAffineInline(e: (op: StackOp) => void, t: ECTracker, c: CurveParams): void { - jacobianAddAffineBody(new ECTracker([...t.nm], e), false, c); + jacobianAddAffineBody(new ECTracker([...t.nm], e, t.options), false, c); } /** @@ -795,7 +796,7 @@ function cSelectCoord( * Stack layout: [..., ax, ay, _k, jx, jy, jz] — same in and out. */ function buildJacobianAddOrDoubleInline(e: (op: StackOp) => void, t: ECTracker, c: CurveParams): void { - const it = new ECTracker([...t.nm], e); + const it = new ECTracker([...t.nm], e, t.options); // Keep the pre-add accumulator: it is what must be DOUBLED in the // exceptional case, and the add below consumes jx/jy/jz. @@ -862,8 +863,11 @@ function cEmitMul( emit: (op: StackOp) => void, c: CurveParams, g: GroupParams, + opts?: EcCodegenOptions, ): void { - const t = new ECTracker(['_pt', '_k'], emit); + const t = new ECTracker(['_pt', '_k'], emit, opts); + t.poolConstant(POOL_FIELD_P, c.fieldP); + t.poolConstant(POOL_GROUP_N, g.n); cDecomposePoint(t, '_pt', 'ax', 'ay', c); // k' = k + 3n: guarantees a fixed high bit for MSB-first double-and-add. @@ -943,6 +947,8 @@ function cEmitMul( t.toTop('_k'); t.drop(); cComposePoint(t, '_rx', '_ry', '_result', c); + t.releaseConstant(POOL_GROUP_N); + t.releaseConstant(POOL_FIELD_P); } // =========================================================================== @@ -1298,8 +1304,15 @@ function cEmitVerifyECDSA( sqrtExp: bigint, gx: bigint, gy: bigint, + opts?: EcCodegenOptions, ): void { - const t = new ECTracker(['_msg', '_sig', '_pk'], emit); + const t = new ECTracker(['_msg', '_sig', '_pk'], emit, opts); + // The verifier does hundreds of reductions OUTSIDE the two ladders — + // decompression's sqrt ladder, cGroupInv, cAffineAdd, the final cGroupMod. + // Each ladder pools separately: cEmitMul runs on its own tracker that + // deliberately cannot see this stack, so it cannot reach this slot. + t.poolConstant(POOL_FIELD_P, c.fieldP); + t.poolConstant(POOL_GROUP_N, g.n); // Step 0: length gate. `_sig` and `_pk` are bare ByteString in the builtin // table and the type checker imposes no width, so both arrive attacker-sized. @@ -1410,7 +1423,7 @@ function cEmitVerifyECDSA( t.nm.pop(); // _G // Emit the mul (it manages its own tracker internally) - cEmitMul(emit, c, g); + cEmitMul(emit, c, g, opts); // After mul, one result point is on the stack t.nm.push('_R1_point'); @@ -1433,7 +1446,7 @@ function cEmitVerifyECDSA( // Pop from tracker, emit mul, push result t.nm.pop(); // _u2 t.nm.pop(); // _Q_point - cEmitMul(emit, c, g); + cEmitMul(emit, c, g, opts); t.nm.push('_R2_point'); // Restore R1 point @@ -1487,6 +1500,8 @@ function cEmitVerifyECDSA( t.rawBlock(['_input_ok', '_sig_ok'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_GROUP_N); + t.releaseConstant(POOL_FIELD_P); } // =========================================================================== @@ -1498,12 +1513,14 @@ function cEmitVerifyECDSA( * Stack in: [P256Point, P256Point] (second on top) * Stack out: [P256Point] */ -export function emitP256Add(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pa', '_pb'], emit); +export function emitP256Add(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pa', '_pb'], emit, opts); + t.poolConstant(POOL_FIELD_P, P256_PARAMS.fieldP); cDecomposePoint(t, '_pa', 'px', 'py', P256_PARAMS); cDecomposePoint(t, '_pb', 'qx', 'qy', P256_PARAMS); cAffineAdd(t, P256_PARAMS); cComposePoint(t, 'rx', 'ry', '_result', P256_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1511,8 +1528,8 @@ export function emitP256Add(emit: (op: StackOp) => void): void { * Stack in: [P256Point, bigint] (scalar on top) * Stack out: [P256Point] */ -export function emitP256Mul(emit: (op: StackOp) => void): void { - cEmitMul(emit, P256_PARAMS, P256_GROUP); +export function emitP256Mul(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitMul(emit, P256_PARAMS, P256_GROUP, opts); } /** @@ -1520,13 +1537,13 @@ export function emitP256Mul(emit: (op: StackOp) => void): void { * Stack in: [bigint] * Stack out: [P256Point] */ -export function emitP256MulGen(emit: (op: StackOp) => void): void { +export function emitP256MulGen(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { const gPoint = new Uint8Array(64); gPoint.set(bigintToBytes(P256_GX, 32), 0); gPoint.set(bigintToBytes(P256_GY, 32), 32); emit({ op: 'push', value: gPoint }); emit({ op: 'swap' }); // [point, scalar] - emitP256Mul(emit); + emitP256Mul(emit, opts); } /** @@ -1534,12 +1551,14 @@ export function emitP256MulGen(emit: (op: StackOp) => void): void { * Stack in: [P256Point] * Stack out: [P256Point] */ -export function emitP256Negate(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP256Negate(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P256_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_nx', '_ny', P256_PARAMS); pushFieldP(t, '_fp', P256_PARAMS); cFieldSub(t, '_fp', '_ny', '_neg_y', P256_PARAMS); cComposePoint(t, '_nx', '_neg_y', '_result', P256_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1547,8 +1566,9 @@ export function emitP256Negate(emit: (op: StackOp) => void): void { * Stack in: [P256Point] * Stack out: [boolean] */ -export function emitP256OnCurve(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP256OnCurve(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P256_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_x', '_y', P256_PARAMS); cEmitCanonicityGuard(t, '_x', '_y', P256_PARAMS); @@ -1578,6 +1598,7 @@ export function emitP256OnCurve(emit: (op: StackOp) => void): void { t.rawBlock(['_canon', '_curve_eq'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1616,8 +1637,8 @@ export function emitP256EncodeCompressed(emit: (op: StackOp) => void): void { * Stack in: [msg_bytes, sig(64B), pubkey(33B)] (pubkey on top) * Stack out: [boolean] */ -export function emitVerifyECDSA_P256(emit: (op: StackOp) => void): void { - cEmitVerifyECDSA(emit, P256_PARAMS, P256_GROUP, P256_B, P256_SQRT_EXP, P256_GX, P256_GY); +export function emitVerifyECDSA_P256(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitVerifyECDSA(emit, P256_PARAMS, P256_GROUP, P256_B, P256_SQRT_EXP, P256_GX, P256_GY, opts); } // =========================================================================== @@ -1629,12 +1650,14 @@ export function emitVerifyECDSA_P256(emit: (op: StackOp) => void): void { * Stack in: [P384Point, P384Point] (second on top) * Stack out: [P384Point] */ -export function emitP384Add(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pa', '_pb'], emit); +export function emitP384Add(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pa', '_pb'], emit, opts); + t.poolConstant(POOL_FIELD_P, P384_PARAMS.fieldP); cDecomposePoint(t, '_pa', 'px', 'py', P384_PARAMS); cDecomposePoint(t, '_pb', 'qx', 'qy', P384_PARAMS); cAffineAdd(t, P384_PARAMS); cComposePoint(t, 'rx', 'ry', '_result', P384_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1642,8 +1665,8 @@ export function emitP384Add(emit: (op: StackOp) => void): void { * Stack in: [P384Point, bigint] (scalar on top) * Stack out: [P384Point] */ -export function emitP384Mul(emit: (op: StackOp) => void): void { - cEmitMul(emit, P384_PARAMS, P384_GROUP); +export function emitP384Mul(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitMul(emit, P384_PARAMS, P384_GROUP, opts); } /** @@ -1651,13 +1674,13 @@ export function emitP384Mul(emit: (op: StackOp) => void): void { * Stack in: [bigint] * Stack out: [P384Point] */ -export function emitP384MulGen(emit: (op: StackOp) => void): void { +export function emitP384MulGen(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { const gPoint = new Uint8Array(96); gPoint.set(bigintToBytes(P384_GX, 48), 0); gPoint.set(bigintToBytes(P384_GY, 48), 48); emit({ op: 'push', value: gPoint }); emit({ op: 'swap' }); // [point, scalar] - emitP384Mul(emit); + emitP384Mul(emit, opts); } /** @@ -1665,12 +1688,14 @@ export function emitP384MulGen(emit: (op: StackOp) => void): void { * Stack in: [P384Point] * Stack out: [P384Point] */ -export function emitP384Negate(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP384Negate(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P384_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_nx', '_ny', P384_PARAMS); pushFieldP(t, '_fp', P384_PARAMS); cFieldSub(t, '_fp', '_ny', '_neg_y', P384_PARAMS); cComposePoint(t, '_nx', '_neg_y', '_result', P384_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1678,8 +1703,9 @@ export function emitP384Negate(emit: (op: StackOp) => void): void { * Stack in: [P384Point] * Stack out: [boolean] */ -export function emitP384OnCurve(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP384OnCurve(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P384_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_x', '_y', P384_PARAMS); cEmitCanonicityGuard(t, '_x', '_y', P384_PARAMS); @@ -1709,6 +1735,7 @@ export function emitP384OnCurve(emit: (op: StackOp) => void): void { t.rawBlock(['_canon', '_curve_eq'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1747,6 +1774,6 @@ export function emitP384EncodeCompressed(emit: (op: StackOp) => void): void { * Stack in: [msg_bytes, sig(96B), pubkey(49B)] (pubkey on top) * Stack out: [boolean] */ -export function emitVerifyECDSA_P384(emit: (op: StackOp) => void): void { - cEmitVerifyECDSA(emit, P384_PARAMS, P384_GROUP, P384_B, P384_SQRT_EXP, P384_GX, P384_GY); +export function emitVerifyECDSA_P384(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitVerifyECDSA(emit, P384_PARAMS, P384_GROUP, P384_B, P384_SQRT_EXP, P384_GX, P384_GY, opts); } diff --git a/packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts b/packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts new file mode 100644 index 000000000..7ab6abad3 --- /dev/null +++ b/packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts @@ -0,0 +1,251 @@ +/** + * EC constant pooling — semantic equivalence on the real interpreter. + * + * Pooling replaces ~20,000 inline pushes of a curve's field prime with picks + * from one resident stack slot (see + * `docs/experiments/script-size-optimization-baseline.md`). It changes stack + * layout inside every EC emitter, including inside `OP_IF` arms, so "the byte + * count went down" is not evidence of anything on its own. + * + * Two kinds of proof here, both through @bsv/sdk's `Spend`: + * + * 1. DIFFERENTIAL — for the same inputs, the pooled and unpooled scripts leave + * an identical stack. No oracle needed and no fixture to get wrong: the + * unpooled emitter is the specification. + * 2. ORACLE — `verifyECDSA_*` accepts a genuine OpenSSL signature and rejects + * every near-miss, under BOTH variants. This is the one that would catch a + * pooled slot being read where a *different* value was intended, which a + * pure differential over random inputs can miss if both variants are wrong + * in the same way (they cannot be here — only one of them was changed — + * but the reject cases also pin the security-relevant behaviour). + * + * Max stack depth is measured, not assumed: pooling adds resident slots, and + * the interpreter's 1,000-element budget is the real limit. + */ + +import { describe, it, expect } from 'vitest'; +import { createSign, generateKeyPairSync } from 'node:crypto'; +import { + emitMethod, + emitVerifyECDSA_P256, emitVerifyECDSA_P384, + emitP256Add, emitP256Mul, emitP256Negate, emitP256OnCurve, + emitP384Add, emitP384Negate, emitP384OnCurve, + emitEcAdd, emitEcMul, emitEcNegate, emitEcOnCurve, +} from 'runar-compiler'; +import type { StackOp } from 'runar-ir-schema'; +import { ScriptVM } from '../index.js'; + +type Emitter = (emit: (op: StackOp) => void, opts?: { constantPool?: boolean }) => void; + +const blob = (hex: string) => Uint8Array.from(Buffer.from(hex, 'hex')); + +interface RunResult { + stack: string[]; + error: string | null; + maxStackDepth: number; +} + +/** Emit `inputs` then the emitter's body, and execute the whole thing. */ +function run(emitter: Emitter, inputs: StackOp[], pooled: boolean): RunResult { + const ops: StackOp[] = [...inputs]; + emitter(op => ops.push(op), pooled ? { constantPool: true } : undefined); + const { scriptHex } = emitMethod({ name: 't', ops } as never) as { scriptHex: string }; + const r = new ScriptVM().executeHex(scriptHex) as never as { + stack: Uint8Array[]; error?: string; maxStackDepth: number; + }; + return { + stack: r.stack.map(b => Buffer.from(b).toString('hex')), + error: r.error ?? null, + maxStackDepth: r.maxStackDepth, + }; +} + +/** Assert both variants agree completely, and report the depth cost. */ +function expectSame(emitter: Emitter, inputs: StackOp[]): { off: RunResult; on: RunResult } { + const off = run(emitter, inputs, false); + const on = run(emitter, inputs, true); + expect(on.error).toBe(off.error); + expect(on.stack).toEqual(off.stack); + return { off, on }; +} + +const push = (hex: string): StackOp => ({ op: 'push', value: blob(hex) } as StackOp); +const pushN = (n: bigint): StackOp => ({ op: 'push', value: n } as StackOp); + +// --------------------------------------------------------------------------- +// Curve fixtures +// --------------------------------------------------------------------------- + +const P256 = { + p: 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn, + n: 0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551n, + gx: 0x6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296n, + gy: 0x4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5n, + bytes: 32, +}; +const SECP = { + p: 0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2fn, + n: 0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141n, + gx: 0x79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798n, + gy: 0x483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8n, + bytes: 32, +}; +const P384 = { + gx: 0xaa87ca22be8b05378eb1c71ef320ad746e1d3b628ba79b9859f741e082542a385502f25dbf55296c3a545e3872760ab7n, + gy: 0x3617de4a96262c6f5d9e98bf9292dc29f8f41dbd289a147ce9da3113b5f0b8c00a60b1ce1d7e819d7a431d7c90ea0e5fn, + bytes: 48, +}; + +const hx = (v: bigint, bytes: number) => v.toString(16).padStart(bytes * 2, '0'); +const point = (x: bigint, y: bigint, bytes: number) => hx(x, bytes) + hx(y, bytes); + +// --------------------------------------------------------------------------- +// 1. Differential — the unpooled emitter is the specification +// --------------------------------------------------------------------------- + +describe('pooled and unpooled emitters agree (differential)', () => { + const G256 = point(P256.gx, P256.gy, 32); + const GSEC = point(SECP.gx, SECP.gy, 32); + const G384 = point(P384.gx, P384.gy, 48); + + it('p256Add: G + G (the doubling path)', () => { + expectSame(emitP256Add, [push(G256), push(G256)]); + }); + + it('p256Negate: -G', () => { + expectSame(emitP256Negate, [push(G256)]); + }); + + it('p256OnCurve: accepts G', () => { + const { off } = expectSame(emitP256OnCurve, [push(G256)]); + expect(off.stack).toEqual(['01']); + }); + + it('p256OnCurve: rejects a point off the curve', () => { + const { off } = expectSame(emitP256OnCurve, [push(point(P256.gx, P256.gy + 1n, 32))]); + expect(off.stack).toEqual(['']); + }); + + it('p256OnCurve: rejects a non-canonical x >= p', () => { + // The pooled prime is what the canonicity guard compares against, so this + // is the case that would break first if the pool ever served a stale slot. + expectSame(emitP256OnCurve, [push(point(P256.gx + P256.p, P256.gy, 32))]); + }); + + it.each([1n, 2n, 3n, 7n, P256.n - 1n, 0n, P256.n])('p256Mul: G * %s', (k) => { + expectSame(emitP256Mul, [push(G256), pushN(k)]); + }); + + it('p384Add: G + G', () => { + expectSame(emitP384Add, [push(G384), push(G384)]); + }); + + it('p384Negate / p384OnCurve on G', () => { + expectSame(emitP384Negate, [push(G384)]); + expectSame(emitP384OnCurve, [push(G384)]); + }); + + it('ecAdd: G + G (secp256k1)', () => { + expectSame(emitEcAdd, [push(GSEC), push(GSEC)]); + }); + + it('ecNegate / ecOnCurve on G (secp256k1)', () => { + expectSame(emitEcNegate, [push(GSEC)]); + expectSame(emitEcOnCurve, [push(GSEC)]); + }); + + it.each([1n, 2n, 5n, SECP.n - 1n, 0n])('ecMul: G * %s (secp256k1)', (k) => { + expectSame(emitEcMul, [push(GSEC), pushN(k)]); + }); + + it('agrees on garbage inputs too — both must fail the same way', () => { + // Totality matters: these builtins are specified as "consume N, push 1" + // for ANY argument bytes, so a divergence in the ERROR is as bad as a + // divergence in the result. + expectSame(emitP256OnCurve, [push('00'.repeat(64))]); + expectSame(emitP256Add, [push('ff'.repeat(64)), push('00'.repeat(64))]); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Oracle — OpenSSL signatures, both variants +// --------------------------------------------------------------------------- + +/** DER SEQUENCE { INTEGER r, INTEGER s } -> fixed-width r||s. */ +function derToRaw(der: Buffer, bytes: number): string { + let i = 0; + if (der[i++] !== 0x30) throw new Error('not a DER sequence'); + if (der[i]! & 0x80) i += 1 + (der[i]! & 0x7f); else i += 1; + const readInt = (): bigint => { + if (der[i++] !== 0x02) throw new Error('not a DER integer'); + const len = der[i++]!; + const v = BigInt('0x' + der.subarray(i, i + len).toString('hex')); + i += len; + return v; + }; + const r = readInt(); + const s = readInt(); + const w = bytes * 2; + return r.toString(16).padStart(w, '0') + s.toString(16).padStart(w, '0'); +} + +const CURVES = [ + { name: 'p256', node: 'prime256v1' as const, bytes: 32, emit: emitVerifyECDSA_P256, n: P256.n }, + { name: 'p384', node: 'secp384r1' as const, bytes: 48, emit: emitVerifyECDSA_P384, + n: 0xffffffffffffffffffffffffffffffffffffffffffffffffc7634d81f4372ddf581a0db248b0a77aecec196accc52973n }, +]; + +for (const c of CURVES) { + describe(`${c.name} verifyECDSA agrees under pooling (OpenSSL oracle)`, () => { + const { privateKey, publicKey } = generateKeyPairSync('ec', { namedCurve: c.node }); + const pub = publicKey.export({ format: 'der', type: 'spki' }) as Buffer; + const uncompressed = pub.subarray(pub.length - (1 + c.bytes * 2)).toString('hex'); + const w = c.bytes * 2; + const qx = BigInt('0x' + uncompressed.slice(2, 2 + w)); + const qy = BigInt('0x' + uncompressed.slice(2 + w)); + const compressed = ((qy & 1n) === 0n ? '02' : '03') + hx(qx, c.bytes); + + const msgHex = '52c3ad6172206d657373616765'; // "Rúnar message" + const signer = createSign('sha256'); + signer.update(Buffer.from(msgHex, 'hex')); + const sigHex = derToRaw(signer.sign(privateKey) as Buffer, c.bytes); + + const verify = (msg: string, sig: string, pk: string, pooled: boolean): boolean => { + const r = run(c.emit, [push(msg), push(sig), push(pk)], pooled); + expect(r.error, 'verifier aborted instead of returning a boolean').toBe(null); + expect(r.stack.length, 'specified as 3 args in, 1 boolean out').toBe(1); + return r.stack[0] !== '' && r.stack[0] !== '00'; + }; + + const zero = '0'.repeat(w); + const rGen = sigHex.slice(0, w); + const sGen = sigHex.slice(w); + const flipped = (compressed.slice(0, 2) === '02' ? '03' : '02') + compressed.slice(2); + + const CASES: Array<[string, string, string, string, boolean]> = [ + ['genuine signature', msgHex, sigHex, compressed, true], + ['wrong message', msgHex + '00', sigHex, compressed, false], + ['wrong pubkey parity', msgHex, sigHex, flipped, false], + ['all-zero signature (universal forgery)', msgHex, zero + zero, compressed, false], + ['r = 0', msgHex, zero + sGen, compressed, false], + ['s = 0', msgHex, rGen + zero, compressed, false], + ['r = n', msgHex, hx(c.n, c.bytes) + sGen, compressed, false], + ['s = n', msgHex, rGen + hx(c.n, c.bytes), compressed, false], + ['truncated signature', msgHex, sigHex.slice(0, w), compressed, false], + ['oversized signature', msgHex, sigHex + 'ff', compressed, false], + ]; + + it.each(CASES)('%s', (_label, msg, sig, pk, want) => { + expect(verify(msg, sig, pk, false)).toBe(want); + expect(verify(msg, sig, pk, true)).toBe(want); + }); + + it('does not blow the interpreter stack budget', () => { + const off = run(c.emit, [push(msgHex), push(sigHex), push(compressed)], false); + const on = run(c.emit, [push(msgHex), push(sigHex), push(compressed)], true); + // The pool is a small constant number of extra resident slots. + expect(on.maxStackDepth).toBeLessThanOrEqual(off.maxStackDepth + 8); + expect(on.maxStackDepth).toBeLessThan(800); + }); + }); +} diff --git a/packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts b/packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts new file mode 100644 index 000000000..b6eea482b --- /dev/null +++ b/packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts @@ -0,0 +1,144 @@ +/** + * Liveness scheduler + EC constant pool — semantic equivalence, source vs script. + * + * Both experiments move values around the stack (the scheduler parks results + * on the alt stack; the pool serves a constant from a resident slot). That is + * exactly the class of change that can produce a script which still runs and + * still leaves a truthy top-of-stack while computing something else, so a byte + * count proves nothing on its own. + * + * `runDifferentialExecution` compiles a contract, executes the deployed script + * on the real @bsv/sdk engine, AND runs the same spend through the ANF + * interpreter — a source-semantics oracle that knows nothing about stack + * layout. Running it once per variant on the same witness gives translation + * validation: + * + * current(w) == variant(w) == interpreter(w) for every witness w + * + * The witnesses are NOT invented here: they come from `conformance/witnesses/`, + * the same specs CI already runs, so every expectation is one the repo has + * independently committed to. `conformance/witnesses/coverage-claims.test.ts` + * enforces that each spec carries at least one accept AND one reject, so a + * variant that made everything pass cannot slip through. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { runDifferentialExecution, type WitnessArg } from '../oracle/differential-execution.js'; + +const CONFORMANCE = resolve(__dirname, '../../../../conformance'); +const WITNESS_DIR = join(CONFORMANCE, 'witnesses'); +const TESTS_DIR = join(CONFORMANCE, 'tests'); + +/** Decode a method-argument literal: bigint ("27n"), boolean, or bytes ("0x…"). */ +function decodeArg(v: unknown): WitnessArg { + if (typeof v === 'boolean') return v; + if (typeof v === 'string') { + if (/^-?\d+n$/.test(v)) return BigInt(v.slice(0, -1)); + if (v.startsWith('0x')) return Uint8Array.from(Buffer.from(v.slice(2), 'hex')); + } + throw new Error(`unencodable witness arg: ${JSON.stringify(v)}`); +} + +function decodeCtor(v: unknown): bigint | boolean | string { + if (typeof v === 'boolean') return v; + if (typeof v === 'string') { + if (/^-?\d+n$/.test(v)) return BigInt(v.slice(0, -1)); + if (v.startsWith('0x')) return v.slice(2); + } + throw new Error(`unencodable constructor arg: ${JSON.stringify(v)}`); +} + +interface Spend { method: string; args: unknown[]; expect: 'accept' | 'reject'; note?: string } +interface Spec { fixture: string; constructorArgs?: Record; spends: Spend[] } + +const NON_SPEC_JSON = new Set(['coverage-ledger.json']); +const SPECS: Spec[] = readdirSync(WITNESS_DIR) + .filter(f => f.endsWith('.json') && !NON_SPEC_JSON.has(f)) + .sort() + .map(f => JSON.parse(readFileSync(join(WITNESS_DIR, f), 'utf-8')) as Spec); + +/** Variants under test. `current` is the baseline every other is compared to. */ +const VARIANTS = [ + { name: 'liveness', opts: { schedulerMode: 'liveness' as const } }, + { name: 'ec-pool', opts: { ecConstantPool: true } }, + { name: 'both', opts: { schedulerMode: 'liveness' as const, ecConstantPool: true } }, +]; + +describe('experimental backends preserve acceptance', () => { + it('found the witness corpus', () => { + expect(SPECS.length).toBeGreaterThanOrEqual(10); + }); + + for (const spec of SPECS) { + const fixtureDir = join(TESTS_DIR, spec.fixture); + const srcCfg = JSON.parse(readFileSync(join(fixtureDir, 'source.json'), 'utf-8')) as + { sources?: Record; path?: string }; + const tsRel = srcCfg.sources?.['.runar.ts'] ?? srcCfg.path; + if (!tsRel) throw new Error(`no .runar.ts source in ${spec.fixture}/source.json`); + const srcPath = resolve(fixtureDir, tsRel); + const source = readFileSync(srcPath, 'utf-8'); + const fileName = srcPath.split('/').pop()!; + const ctor: Record = {}; + for (const [k, v] of Object.entries(spec.constructorArgs ?? {})) ctor[k] = decodeCtor(v); + + describe(spec.fixture, () => { + for (const s of spec.spends) { + for (const variant of VARIANTS) { + it(`${variant.name}: ${s.method}(${s.args.join(',')}) → ${s.expect}`, () => { + const common = { + source, fileName, method: s.method, + args: s.args.map(decodeArg), constructorArgs: ctor, + }; + const base = runDifferentialExecution(common); + const other = runDifferentialExecution({ ...common, ...variant.opts }); + + // The witness spec itself must hold, or the comparison is vacuous. + expect(base.vmAccepted, 'witness spec disagrees with the shipping compiler') + .toBe(s.expect === 'accept'); + // Translation validation, both directions against the interpreter. + expect(other.vmAccepted).toBe(base.vmAccepted); + expect(base.vmAccepted).toBe(base.interpreterAccepted); + expect(other.vmAccepted).toBe(other.interpreterAccepted); + expect(other.vmError ?? null).toBe(base.vmError ?? null); + }); + } + } + }); + } + + it('the liveness scheduler really does change bytes somewhere in this corpus', () => { + // Guards against the whole suite passing because every variant compiled to + // the identical script. + const changed: string[] = []; + for (const spec of SPECS) { + const fixtureDir = join(TESTS_DIR, spec.fixture); + const srcCfg = JSON.parse(readFileSync(join(fixtureDir, 'source.json'), 'utf-8')) as + { sources?: Record; path?: string }; + const tsRel = srcCfg.sources?.['.runar.ts'] ?? srcCfg.path; + if (!tsRel) continue; + const srcPath = resolve(fixtureDir, tsRel); + const ctor: Record = {}; + for (const [k, v] of Object.entries(spec.constructorArgs ?? {})) ctor[k] = decodeCtor(v); + const first = spec.spends[0]!; + const common = { + source: readFileSync(srcPath, 'utf-8'), + fileName: srcPath.split('/').pop()!, + method: first.method, + args: first.args.map(decodeArg), + constructorArgs: ctor, + }; + const base = runDifferentialExecution(common); + const sched = runDifferentialExecution({ ...common, schedulerMode: 'liveness' }); + if (sched.lockingHex !== base.lockingHex) { + changed.push(spec.fixture); + // And it must never be bigger — the cost model picks per method. + expect(sched.lockingHex.length, `${spec.fixture} grew`) + .toBeLessThan(base.lockingHex.length); + } + } + expect(changed.length, 'scheduler was a no-op on every witnessed fixture').toBeGreaterThan(0); + console.log(` scheduler changed bytes on: ${changed.join(', ')}`); + }); +}); diff --git a/packages/runar-testing/src/__tests__/scheduler-headroom.test.ts b/packages/runar-testing/src/__tests__/scheduler-headroom.test.ts new file mode 100644 index 000000000..f8e4c2e21 --- /dev/null +++ b/packages/runar-testing/src/__tests__/scheduler-headroom.test.ts @@ -0,0 +1,127 @@ +/** + * Headroom probe for the stack scheduler. + * + * `conformance/tests/arithmetic` is the smallest fixture whose bytes are + * produced ENTIRELY by the generic ANF -> Stack lowering: no crypto macro, no + * sighash scaffolding, no state continuation. 16 of its 28 bytes (57 %) are + * stack access. That makes it the honest measuring stick for "how much can a + * better schedule win on ordinary contracts?". + * + * This test pins two things: + * + * 1. what the compiler emits today, and + * 2. that a hand-written alternative schedule — operands held hot at the top, + * finished results parked on the alt stack — accepts and rejects exactly + * the same inputs while being materially smaller. + * + * (2) is not a claim about what the compiler does; it is the TARGET the + * liveness scheduler is aimed at, executed on the real interpreter so the + * headroom number in `docs/experiments/stack-scheduler-design.md` is measured + * rather than estimated. If a future scheduler beats it, tighten this test. + */ + +import { describe, it, expect } from 'vitest'; +import { ScriptVM } from '../vm/script-vm.js'; + +/** Encode a bigint as a minimally-encoded Bitcoin script number push. */ +function pushNum(n: bigint): string { + if (n === 0n) return '00'; + if (n >= 1n && n <= 16n) return (0x50 + Number(n)).toString(16).padStart(2, '0'); + const neg = n < 0n; + let v = neg ? -n : n; + const bytes: number[] = []; + while (v > 0n) { bytes.push(Number(v & 0xffn)); v >>= 8n; } + if (bytes[bytes.length - 1]! & 0x80) bytes.push(neg ? 0x80 : 0x00); + else if (neg) bytes[bytes.length - 1] = bytes[bytes.length - 1]! | 0x80; + return bytes.length.toString(16).padStart(2, '0') + bytes.map(b => b.toString(16).padStart(2, '0')).join(''); +} + +/** + * What the compiler emits today for `Arithmetic.verify`, with the constructor + * placeholder (`00`) replaced by a real target push. + * + * OP_2DUP OP_ADD sum + * OP_2 OP_PICK OP_2 OP_PICK OP_SUB diff + * OP_3 OP_PICK OP_3 OP_PICK OP_MUL prod + * OP_4 OP_ROLL OP_4 OP_ROLL OP_DIV quot + * OP_3 OP_ROLL OP_3 OP_ROLL OP_ADD OP_ROT OP_ADD OP_SWAP OP_ADD + * OP_NUMEQUAL + */ +function currentSchedule(target: bigint): string { + return `6e9352795279945379537995547a547a96537a537a937b937c93${pushNum(target)}9c`; +} + +/** + * The same computation, scheduled so `a` and `b` never leave the top two + * slots and each finished result is spilled to the alt stack: + * + * OP_2DUP OP_ADD OP_TOALTSTACK sum -> alt + * OP_2DUP OP_SUB OP_TOALTSTACK diff -> alt + * OP_2DUP OP_MUL OP_TOALTSTACK prod -> alt + * OP_DIV quot (consumes a, b) + * OP_FROMALTSTACK OP_ADD + prod + * OP_FROMALTSTACK OP_ADD + diff + * OP_FROMALTSTACK OP_ADD + sum + * OP_NUMEQUAL + * + * Addition is associative and commutative over script numbers here, so the + * reversed accumulation order is value-identical. + */ +function altStackSchedule(target: bigint): string { + return `6e936b6e946b6e956b966c936c936c93${pushNum(target)}9c`; +} + +function run(scriptHex: string, a: bigint, b: bigint): boolean { + const vm = new ScriptVM(); + const unlocking = `${pushNum(a)}${pushNum(b)}`; + const r = vm.execute( + Uint8Array.from(Buffer.from(unlocking, 'hex')), + Uint8Array.from(Buffer.from(scriptHex, 'hex')), + ); + return r.success; +} + +/** a + b, a - b, a * b, a / b summed — the contract's `result`. */ +function expected(a: bigint, b: bigint): bigint { + // Script's OP_DIV truncates toward zero, which matches bigint division. + return (a + b) + (a - b) + a * b + a / b; +} + +const CASES: [bigint, bigint][] = [ + [7n, 3n], [3n, 7n], [1n, 1n], [100n, 7n], [-5n, 3n], [5n, -3n], + [-5n, -3n], [0n, 1n], [16n, 16n], [17n, 2n], [255n, 4n], [-1n, -1n], + [1000n, 3n], [2n, 1000n], +]; + +describe('stack scheduler headroom (conformance/tests/arithmetic)', () => { + it('pins the byte cost of both schedules', () => { + // 5 is the byte cost of the `target` push in these probes (4-byte push of + // a value that needs a sign byte); both schedules carry the same one, so + // the difference is entirely scheduling. + const t = 1000n; + const cur = currentSchedule(t).length / 2; + const alt = altStackSchedule(t).length / 2; + expect(cur).toBe(30); + expect(alt).toBe(20); + // 33 % fewer bytes, all of it stack traffic. + expect(1 - alt / cur).toBeGreaterThan(0.3); + }); + + it('the emitted schedule matches the checked-in golden modulo the placeholder', () => { + // Golden is the template: `00` where the constructor arg is spliced in. + const template = '6e9352795279945379537995547a547a96537a537a937b937c93009c'; + expect(currentSchedule(0n)).toBe(template); + }); + + it.each(CASES)('both schedules accept exactly the right target for a=%s b=%s', (a, b) => { + const want = expected(a, b); + expect(run(currentSchedule(want), a, b)).toBe(true); + expect(run(altStackSchedule(want), a, b)).toBe(true); + }); + + it.each(CASES)('both schedules reject a wrong target for a=%s b=%s', (a, b) => { + const wrong = expected(a, b) + 1n; + expect(run(currentSchedule(wrong), a, b)).toBe(false); + expect(run(altStackSchedule(wrong), a, b)).toBe(false); + }); +}); diff --git a/packages/runar-testing/src/oracle/differential-execution.ts b/packages/runar-testing/src/oracle/differential-execution.ts index 8c68f3ed4..b40becc56 100644 --- a/packages/runar-testing/src/oracle/differential-execution.ts +++ b/packages/runar-testing/src/oracle/differential-execution.ts @@ -50,6 +50,14 @@ export interface DiffExecOptions { args: WitnessArg[]; // method arguments (interpreter + witness order) constructorArgs?: Record; disableConstantFolding?: boolean; // default false → fold-ON deployed bytes + /** + * EXPERIMENTAL backend toggles. Present so a size experiment can be run + * through this same source-vs-script oracle: compile the SAME contract with + * a flag off and on, execute both on the SAME witness, and require identical + * acceptance. Both default to the shipping path. + */ + schedulerMode?: 'current' | 'liveness'; + ecConstantPool?: boolean; } export interface DiffExecResult { @@ -88,6 +96,8 @@ export function runDifferentialExecution(opts: DiffExecOptions): DiffExecResult fileName: opts.fileName, disableConstantFolding: opts.disableConstantFolding ?? false, constructorArgs: ctor, + schedulerMode: opts.schedulerMode, + ecConstantPool: opts.ecConstantPool, }); if (!compiled.success || !compiled.artifact) { const errs = compiled.diagnostics