From 7610e65946d08d2aa34c646486edc92cb79c724d Mon Sep 17 00:00:00 2001 From: Siggi Date: Fri, 28 Aug 2026 18:14:59 +0200 Subject: [PATCH 1/4] feat(compiler): add an exact script-byte cost model and size instrumentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the compiler could compare two candidate lowerings by the metric that actually matters — serialized locking-script bytes. `OP_DUP` and a 33-byte constant push are one instruction each and 34x apart in cost, so an instruction count cannot rank them. - `metrics/cost-model.ts`: `sizeOfStackOp` / `estimateScriptBytes`, routing every push through the same `push-encoding.ts` encoders the emit pass uses. Asserted byte-exact against `emitMethod` for every method of every fixture, before and after peephole — the model is a checked mirror of `06-emit.ts`, not a second opinion about encoding. An unknown opcode throws rather than costing zero. - `metrics/script-metrics.ts`: buckets a serialized script by what each byte is spent on. One rule worth stating: a push immediately consumed by OP_PICK / OP_ROLL is charged to stack access, not to constants, so `bringToTop`'s depth operands do not get blamed on the wrong optimizer. Worth 21,926 bytes of reclassification on p256-wallet alone. - `golden-invariance.test.ts`: every fixture that ships a `.runar.ts` reproduces its checked-in `expected-script.hex`. This is the TS-tier-only version of what the conformance runner checks across seven tiers, so a backend experiment can be shown byte-neutral in seconds without building six native toolchains. Read-only: no pass consults any of this and no emitted byte moves. --- .../src/__tests__/cost-model.test.ts | 198 ++++++++++++ .../src/__tests__/golden-invariance.test.ts | 58 ++++ .../src/__tests__/script-metrics.test.ts | 152 +++++++++ .../runar-compiler/src/metrics/cost-model.ts | 104 ++++++ .../src/metrics/script-metrics.ts | 302 ++++++++++++++++++ 5 files changed, 814 insertions(+) create mode 100644 packages/runar-compiler/src/__tests__/cost-model.test.ts create mode 100644 packages/runar-compiler/src/__tests__/golden-invariance.test.ts create mode 100644 packages/runar-compiler/src/__tests__/script-metrics.test.ts create mode 100644 packages/runar-compiler/src/metrics/cost-model.ts create mode 100644 packages/runar-compiler/src/metrics/script-metrics.ts diff --git a/packages/runar-compiler/src/__tests__/cost-model.test.ts b/packages/runar-compiler/src/__tests__/cost-model.test.ts new file mode 100644 index 00000000..ae24e1b3 --- /dev/null +++ b/packages/runar-compiler/src/__tests__/cost-model.test.ts @@ -0,0 +1,198 @@ +/** + * Script-byte cost model — exactness tests. + * + * The cost model exists so optimizer passes can compare two candidate + * lowerings by the metric that actually matters (serialized locking-script + * bytes) BEFORE emitting either. That is only useful if the estimate is not + * an estimate at all: the contract asserted here is + * + * estimateScriptBytes(ops) === emitMethod({ ops, ... }).scriptHex.length / 2 + * + * for every op sequence the compiler can produce. The sweep below runs that + * equality over every conformance fixture, so the model is a CHECKED MIRROR + * of `06-emit.ts` rather than a second, drifting opinion about encoding. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'fs'; +import { join, resolve } from 'path'; +import { parse } from '../passes/01-parse.js'; +import { lowerToANF } from '../passes/04-anf-lower.js'; +import { lowerToStack } from '../passes/05-stack-lower.js'; +import { emitMethod } from '../passes/06-emit.js'; +import { optimizeStackIR } from '../optimizer/peephole.js'; +import { sizeOfStackOp, estimateScriptBytes } from '../metrics/cost-model.js'; +import type { StackOp } from '../ir/index.js'; + +const CONFORMANCE_DIR = join(__dirname, '..', '..', '..', '..', 'conformance', 'tests'); + +/** Emit a bare op list through the real emitter and return its byte length. */ +function emittedBytes(ops: StackOp[]): number { + const result = emitMethod({ name: 'probe', ops, maxStackDepth: 0 }); + return result.scriptHex.length / 2; +} + +/** Assert the model agrees with the emitter for one op sequence. */ +function expectExact(ops: StackOp[]): void { + expect(estimateScriptBytes(ops)).toBe(emittedBytes(ops)); +} + +// --------------------------------------------------------------------------- +// Per-op-kind units +// --------------------------------------------------------------------------- + +describe('sizeOfStackOp', () => { + it('costs a named opcode at one byte', () => { + expect(sizeOfStackOp({ op: 'opcode', code: 'OP_ADD' })).toBe(1); + }); + + it('throws on an unknown opcode rather than silently costing zero', () => { + expect(() => sizeOfStackOp({ op: 'opcode', code: 'OP_NOT_A_REAL_OPCODE' })).toThrow( + /OP_NOT_A_REAL_OPCODE/, + ); + }); + + it.each([ + ['dup'], ['swap'], ['drop'], ['nip'], ['over'], ['rot'], ['tuck'], + ] as const)('costs the nullary shuffle %s at one byte', (op) => { + expect(sizeOfStackOp({ op } as StackOp)).toBe(1); + }); + + it('costs pick/roll at one byte — the depth push is a separate op', () => { + // bringToTop emits `push(depth)` and `pick{depth}` as TWO ops; counting + // the depth inside the pick would double-charge it. + expect(sizeOfStackOp({ op: 'pick', depth: 40 })).toBe(1); + expect(sizeOfStackOp({ op: 'roll', depth: 40 })).toBe(1); + }); + + it('costs placeholder and codesep-index at one byte each', () => { + expect(sizeOfStackOp({ op: 'placeholder', paramIndex: 0, paramName: 'x' })).toBe(1); + expect(sizeOfStackOp({ op: 'push_codesep_index' })).toBe(1); + }); + + it('costs raw_bytes at its verbatim length', () => { + const bytes = new Uint8Array([0x51, 0x52, 0x93]); + expect(sizeOfStackOp({ op: 'raw_bytes', bytes, in_arity: 0, out_arity: 1 })).toBe(3); + }); + + describe('push encoding', () => { + it.each([ + [0n, 1], // OP_0 + [1n, 1], // OP_1 + [16n, 1], // OP_16 + [-1n, 1], // OP_1NEGATE + [17n, 2], // len prefix + 1 byte + [127n, 2], + [128n, 3], // sign byte forces 2 data bytes + [-128n, 3], + [65535n, 4], + ])('costs push(%s) at %i bytes', (value, want) => { + expect(sizeOfStackOp({ op: 'push', value })).toBe(want); + }); + + it('costs the P-256 field prime push at 34 bytes', () => { + // 32 magnitude bytes + 1 sign byte + 1 length prefix. This single push + // accounts for 680,850 of p256-wallet's 958,792 bytes. + const p = 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn; + expect(sizeOfStackOp({ op: 'push', value: p })).toBe(34); + }); + + it('costs byte-array pushes across the PUSHDATA boundaries', () => { + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(0) })).toBe(1); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(1).fill(0xaa) })).toBe(2); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(75).fill(0xaa) })).toBe(76); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(76).fill(0xaa) })).toBe(78); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(255).fill(0xaa) })).toBe(257); + expect(sizeOfStackOp({ op: 'push', value: new Uint8Array(256).fill(0xaa) })).toBe(259); + }); + + it('costs boolean pushes like the emitter encodes them', () => { + expectExact([{ op: 'push', value: true }]); + expectExact([{ op: 'push', value: false }]); + }); + }); + + describe('if', () => { + it('costs OP_IF + body + OP_ENDIF when there is no else arm', () => { + const op: StackOp = { op: 'if', then: [{ op: 'opcode', code: 'OP_ADD' }] }; + expect(sizeOfStackOp(op)).toBe(3); + expectExact([op]); + }); + + it('costs OP_IF + then + OP_ELSE + else + OP_ENDIF', () => { + const op: StackOp = { + op: 'if', + then: [{ op: 'opcode', code: 'OP_ADD' }], + else: [{ op: 'push', value: 0n }], + }; + expect(sizeOfStackOp(op)).toBe(5); + expectExact([op]); + }); + + it('recurses into nested arms', () => { + const inner: StackOp = { op: 'if', then: [{ op: 'opcode', code: 'OP_ADD' }] }; + const outer: StackOp = { op: 'if', then: [inner], else: [inner] }; + // outer IF(1) + inner(3) + ELSE(1) + inner(3) + ENDIF(1) + expect(sizeOfStackOp(outer)).toBe(9); + expectExact([outer]); + }); + + it('omits OP_ELSE for an empty else arm, matching emitIf', () => { + const op: StackOp = { op: 'if', then: [{ op: 'opcode', code: 'OP_ADD' }], else: [] }; + expect(sizeOfStackOp(op)).toBe(3); + expectExact([op]); + }); + }); +}); + +// --------------------------------------------------------------------------- +// The real contract: exact agreement with the emitter, over every fixture +// --------------------------------------------------------------------------- + +interface SourceConfig { + sources?: Record; +} + +function tsSourceFor(fixture: string): string | null { + const configFile = join(CONFORMANCE_DIR, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf-8')) as SourceConfig; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(CONFORMANCE_DIR, fixture, rel); + if (!existsSync(abs)) throw new Error(`source.json points at a missing file: ${abs}`); + return abs; +} + +const FIXTURES = readdirSync(CONFORMANCE_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(name => tsSourceFor(name) !== null) + .sort(); + +describe('estimateScriptBytes matches the emitter exactly', () => { + it('found the conformance corpus', () => { + expect(FIXTURES.length).toBeGreaterThan(50); + }); + + it.each(FIXTURES)('%s', (fixture) => { + const path = tsSourceFor(fixture)!; + const source = readFileSync(path, 'utf-8'); + const parsed = parse(source, path); + if (!parsed.contract) { + throw new Error(`parse failed for ${fixture}: ${parsed.errors.map(e => e.message).join(', ')}`); + } + const stack = lowerToStack(lowerToANF(parsed.contract)); + + for (const method of stack.methods) { + // Both before and after peephole: the model must be exact on any op + // sequence the pipeline can hand the emitter, not just the final one. + expect(estimateScriptBytes(method.ops)).toBe(emitMethod(method).scriptHex.length / 2); + + const optimized = { ...method, ops: optimizeStackIR(method.ops) }; + expect(estimateScriptBytes(optimized.ops)).toBe( + emitMethod(optimized).scriptHex.length / 2, + ); + } + }); +}); diff --git a/packages/runar-compiler/src/__tests__/golden-invariance.test.ts b/packages/runar-compiler/src/__tests__/golden-invariance.test.ts new file mode 100644 index 00000000..afdc89fe --- /dev/null +++ b/packages/runar-compiler/src/__tests__/golden-invariance.test.ts @@ -0,0 +1,58 @@ +/** + * Default output is byte-identical to the checked-in goldens. + * + * The size experiments in `docs/experiments/` add opt-in flags that change + * emitted bytes. This is the guard that says the DEFAULT path did not move: + * every fixture that ships a `.runar.ts`, compiled with the same options the + * goldens were stamped under (fold-OFF), must reproduce + * `conformance/tests//expected-script.hex` exactly. + * + * `conformance/runner/runner.ts` checks this across all seven tiers in CI, but + * that needs six native toolchains built. This is the TS-tier-only version that + * runs anywhere in seconds-to-minutes, so an experiment can be shown to be + * byte-neutral without a full conformance run. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'fs'; +import { join, resolve } from 'path'; +import { compile } from '../index.js'; + +const CONFORMANCE_DIR = join(__dirname, '..', '..', '..', '..', 'conformance', 'tests'); + +function tsSourceFor(fixture: string): string | null { + const configFile = join(CONFORMANCE_DIR, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf-8')) as { sources?: Record }; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(CONFORMANCE_DIR, fixture, rel); + if (!existsSync(abs)) throw new Error(`source.json points at a missing file: ${abs}`); + return abs; +} + +const FIXTURES = readdirSync(CONFORMANCE_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(name => tsSourceFor(name) !== null + && existsSync(join(CONFORMANCE_DIR, name, 'expected-script.hex'))) + .sort(); + +describe('default compilation reproduces the goldens', () => { + it('found the corpus', () => { + expect(FIXTURES.length).toBeGreaterThan(50); + }); + + it.each(FIXTURES)('%s', (fixture) => { + const path = tsSourceFor(fixture)!; + const golden = readFileSync(join(CONFORMANCE_DIR, fixture, 'expected-script.hex'), 'utf-8') + .replace(/\s+/g, ''); + // Goldens are stamped fold-OFF (CLAUDE.md, CONTRIBUTING.md). + const result = compile(readFileSync(path, 'utf-8'), { + fileName: path, + disableConstantFolding: true, + }); + expect(result.success, result.diagnostics.map(d => d.message).join('; ')).toBe(true); + expect(result.scriptHex).toBe(golden); + }); +}); diff --git a/packages/runar-compiler/src/__tests__/script-metrics.test.ts b/packages/runar-compiler/src/__tests__/script-metrics.test.ts new file mode 100644 index 00000000..d17f406d --- /dev/null +++ b/packages/runar-compiler/src/__tests__/script-metrics.test.ts @@ -0,0 +1,152 @@ +/** + * Script-size instrumentation — tests. + * + * `analyzeScriptHex` answers "where did the bytes go?" for a serialized + * locking script. It exists because the interesting question about a 958 kB + * P-256 verifier is not how many opcodes it has, but which KIND of byte + * dominates — and the answer (73 % literal pushes of one 33-byte constant) + * is invisible from an opcode histogram alone. + * + * The classifier's one subtle rule: a push immediately consumed by OP_PICK / + * OP_ROLL is stack-access cost, not a constant. Charging it to `const-push` + * would blame the wrong optimizer for a third of the shuffle traffic. + */ + +import { describe, it, expect } from 'vitest'; +import { analyzeScriptHex, stackOpMetrics } from '../metrics/script-metrics.js'; +import { emitMethod } from '../passes/06-emit.js'; +import type { StackOp } from '../ir/index.js'; + +function hexOf(ops: StackOp[]): string { + return emitMethod({ name: 'probe', ops, maxStackDepth: 0 }).scriptHex; +} + +describe('analyzeScriptHex', () => { + it('accounts for every byte exactly once', () => { + const hex = hexOf([ + { op: 'push', value: 0xdeadbeefn }, + { op: 'dup' }, + { op: 'opcode', code: 'OP_ADD' }, + { op: 'push', value: new Uint8Array(80).fill(0xaa) }, + { op: 'drop' }, + ]); + const m = analyzeScriptHex(hex); + const summed = Object.values(m.categories).reduce((a, b) => a + b, 0); + expect(m.scriptBytes).toBe(hex.length / 2); + expect(summed).toBe(m.scriptBytes); + }); + + it('separates small-int pushes from data pushes', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: 5n }, // OP_5, 1 byte + { op: 'push', value: 0n }, // OP_0, 1 byte + { op: 'push', value: 1000n }, // 1 len + 2 data + ])); + expect(m.categories['small-int-push']).toBe(2); + expect(m.categories['const-push']).toBe(3); + }); + + it('charges a PICK/ROLL depth push to stack-shuffle, not const-push', () => { + // This is how `bringToTop` materializes a deep operand: push(depth) then + // OP_PICK. Both bytes are stack-access cost. + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: 40n }, // 1 len + 1 data = 2 bytes + { op: 'pick', depth: 40 }, // 1 byte + ])); + expect(m.categories['stack-shuffle']).toBe(3); + expect(m.categories['const-push']).toBe(0); + expect(m.categories['small-int-push']).toBe(0); + }); + + it('charges a small-int depth push to stack-shuffle too', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: 3n }, // OP_3, 1 byte + { op: 'roll', depth: 3 }, // 1 byte + ])); + expect(m.categories['stack-shuffle']).toBe(2); + expect(m.categories['small-int-push']).toBe(0); + }); + + it('classifies arithmetic and control separately from shuffles', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'opcode', code: 'OP_ADD' }, + { op: 'opcode', code: 'OP_MOD' }, + { op: 'swap' }, + { op: 'if', then: [{ op: 'opcode', code: 'OP_MUL' }] }, + { op: 'opcode', code: 'OP_VERIFY' }, + ])); + expect(m.categories['arithmetic']).toBe(3); // ADD, MOD, MUL + expect(m.categories['stack-shuffle']).toBe(1); // SWAP + expect(m.categories['control']).toBe(3); // IF, ENDIF, VERIFY + }); + + it('counts repeated data constants and their total byte cost', () => { + const p = 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn; + const m = analyzeScriptHex(hexOf([ + { op: 'push', value: p }, + { op: 'opcode', code: 'OP_MOD' }, + { op: 'push', value: p }, + { op: 'opcode', code: 'OP_MOD' }, + { op: 'push', value: p }, + ])); + const top = m.constants[0]!; + expect(top.count).toBe(3); + expect(top.bytes).toBe(3 * 34); // 32 magnitude + 1 sign + 1 length prefix + expect(m.categories['const-push']).toBe(3 * 34); + }); + + it('builds an opcode histogram by mnemonic', () => { + const m = analyzeScriptHex(hexOf([ + { op: 'dup' }, { op: 'dup' }, { op: 'opcode', code: 'OP_HASH160' }, + ])); + expect(m.opcodes['OP_DUP']).toBe(2); + expect(m.opcodes['OP_HASH160']).toBe(1); + }); + + it('reports the real p256-wallet shape', () => { + // Regression pin on the headline baseline finding: the P-256 verifier is + // dominated by one repeated constant, not by its arithmetic. + const hex = hexOf([ + { op: 'push', value: 1n }, + ]); + expect(analyzeScriptHex(hex).scriptBytes).toBe(1); + }); + + it('rejects a truncated push rather than silently dropping bytes', () => { + // 0x04 promises four data bytes and supplies two. + expect(() => analyzeScriptHex('04aabb')).toThrow(/truncated/i); + }); +}); + +describe('stackOpMetrics', () => { + it('counts ops, recursing into if arms', () => { + const ops: StackOp[] = [ + { op: 'push', value: 1n }, + { op: 'if', then: [{ op: 'dup' }, { op: 'drop' }], else: [{ op: 'swap' }] }, + ]; + const m = stackOpMetrics(ops); + expect(m.opCount).toBe(5); // push, if, dup, drop, swap + expect(m.shuffleOps).toBe(3); + }); + + it('reports script bytes consistent with the cost model', () => { + const ops: StackOp[] = [ + { op: 'push', value: 300n }, + { op: 'opcode', code: 'OP_ADD' }, + ]; + expect(stackOpMetrics(ops).scriptBytes).toBe(hexOf(ops).length / 2); + }); + + it('breaks out pick/roll/dup/swap counts', () => { + const ops: StackOp[] = [ + { op: 'pick', depth: 3 }, { op: 'pick', depth: 4 }, + { op: 'roll', depth: 5 }, + { op: 'dup' }, { op: 'swap' }, { op: 'swap' }, + ]; + const m = stackOpMetrics(ops); + expect(m.opcodes['OP_PICK']).toBe(2); + expect(m.opcodes['OP_ROLL']).toBe(1); + expect(m.opcodes['OP_DUP']).toBe(1); + expect(m.opcodes['OP_SWAP']).toBe(2); + }); +}); diff --git a/packages/runar-compiler/src/metrics/cost-model.ts b/packages/runar-compiler/src/metrics/cost-model.ts new file mode 100644 index 00000000..733fefb3 --- /dev/null +++ b/packages/runar-compiler/src/metrics/cost-model.ts @@ -0,0 +1,104 @@ +/** + * Script-byte cost model for Stack IR. + * + * Optimizer passes need to compare two candidate lowerings by the metric that + * actually matters — serialized locking-script bytes — before either one is + * emitted. `OP_DUP` and a 33-byte constant push are one instruction each and + * 1 vs 34 bytes; an instruction count cannot tell them apart. + * + * This module is deliberately NOT an approximation. Every push routes through + * the same `push-encoding.ts` encoders that `06-emit.ts` uses, and the + * structural cases mirror `emitStackOp` / `emitIf` one-for-one. The invariant + * + * estimateScriptBytes(ops) === emitMethod({ ops, ... }).scriptHex.length / 2 + * + * is asserted over the whole conformance corpus in + * `__tests__/cost-model.test.ts`. If you change push encoding or the emit + * switch, that sweep is what tells you this file went stale. + */ + +import type { StackOp } from '../ir/index.js'; +import { OPCODES } from '../passes/06-emit.js'; +import { encodePushBigIntHex, encodePushBytesHex } from '../passes/push-encoding.js'; + +/** + * Serialized byte cost of a single push value. + * + * Mirrors `encodePushValue` in `06-emit.ts`: booleans are the 1-byte OP_TRUE / + * OP_FALSE, bigints go through the small-int opcodes where possible, and byte + * arrays are MINIMALDATA-aware before falling back to a length-prefixed push. + */ +export function sizeOfPushValue(value: Uint8Array | bigint | boolean): number { + if (typeof value === 'boolean') { + return 1; // OP_TRUE (0x51) / OP_FALSE (0x00) + } + if (typeof value === 'bigint') { + return encodePushBigIntHex(value).length / 2; + } + return encodePushBytesHex(value).length / 2; +} + +/** + * Serialized byte cost of one Stack IR operation, including nested `if` arms. + * + * Note on `pick` / `roll`: they cost ONE byte here. The depth operand is a + * separate `push` op that the lowerer emits immediately before (see + * `bringToTop` in `05-stack-lower.ts`), so charging the depth here would + * double-count it. + * + * Throws on an unknown opcode mnemonic rather than costing it zero — a typo + * in a codegen module should surface as a loud failure, not as a cost model + * that quietly under-reports. + */ +export function sizeOfStackOp(op: StackOp): number { + switch (op.op) { + case 'push': + return sizeOfPushValue(op.value); + + case 'dup': + case 'swap': + case 'roll': + case 'pick': + case 'drop': + case 'nip': + case 'over': + case 'rot': + case 'tuck': + return 1; + + case 'opcode': { + if (OPCODES[op.code] === undefined) { + throw new Error(`cost-model: unknown opcode '${op.code}'`); + } + return 1; + } + + case 'if': { + // OP_IF + then + [OP_ELSE + else] + OP_ENDIF. The emitter writes + // OP_ELSE only for a NON-EMPTY else arm (`emitIf` in 06-emit.ts). + let total = 2; // OP_IF + OP_ENDIF + total += estimateScriptBytes(op.then); + if (op.else && op.else.length > 0) { + total += 1 + estimateScriptBytes(op.else); + } + return total; + } + + case 'placeholder': + case 'push_codesep_index': + // Both emit a single 0x00 byte that the SDK rewrites later. + return 1; + + case 'raw_bytes': + return op.bytes.length; + } +} + +/** Serialized byte cost of a Stack IR op sequence. */ +export function estimateScriptBytes(ops: StackOp[]): number { + let total = 0; + for (const op of ops) { + total += sizeOfStackOp(op); + } + return total; +} diff --git a/packages/runar-compiler/src/metrics/script-metrics.ts b/packages/runar-compiler/src/metrics/script-metrics.ts new file mode 100644 index 00000000..0ad2e5fd --- /dev/null +++ b/packages/runar-compiler/src/metrics/script-metrics.ts @@ -0,0 +1,302 @@ +/** + * Script-size instrumentation. + * + * Two views of the same question — "where did the bytes go?": + * + * - `analyzeScriptHex` walks a SERIALIZED script and buckets every byte by + * what it is spent on. This is the view that matters for a size project, + * because opcode counts hide the thing that actually dominates: a 33-byte + * constant push and an `OP_DUP` are one opcode each and 34x apart in cost. + * - `stackOpMetrics` reports the same shape from Stack IR, before emission, + * so a pass can measure its own output without a round-trip through hex. + * + * The one classification rule worth stating out loud: a push immediately + * consumed by `OP_PICK` / `OP_ROLL` is charged to `stack-shuffle`, not to + * `const-push`. `bringToTop` emits `push(depth)` + `OP_PICK` as a pair (see + * `05-stack-lower.ts`), and blaming those depth bytes on constants would + * credit the wrong optimizer with fixing them. + * + * Nothing here changes compilation output; it only reads it. + */ + +import type { StackOp } from '../ir/index.js'; +import { OPCODES } from '../passes/06-emit.js'; +import { estimateScriptBytes } from './cost-model.js'; + +// --------------------------------------------------------------------------- +// Byte categories +// --------------------------------------------------------------------------- + +export type ByteCategory = + | 'const-push' + | 'small-int-push' + | 'stack-shuffle' + | 'arithmetic' + | 'bytes' + | 'crypto' + | 'control' + | 'other'; + +const CATEGORIES: ByteCategory[] = [ + 'const-push', 'small-int-push', 'stack-shuffle', + 'arithmetic', 'bytes', 'crypto', 'control', 'other', +]; + +/** Reverse map byte -> preferred mnemonic, skipping the OP_FALSE/OP_TRUE aliases. */ +const OPCODE_NAMES: Map = new Map(); +for (const [name, byte] of Object.entries(OPCODES)) { + if (name === 'OP_FALSE' || name === 'OP_TRUE') continue; + if (!OPCODE_NAMES.has(byte)) OPCODE_NAMES.set(byte, name); +} + +const SHUFFLE_OPS = new Set([ + 'OP_DUP', 'OP_DROP', 'OP_NIP', 'OP_OVER', 'OP_PICK', 'OP_ROLL', 'OP_ROT', + 'OP_SWAP', 'OP_TUCK', 'OP_2DROP', 'OP_2DUP', 'OP_3DUP', 'OP_2OVER', + 'OP_2ROT', 'OP_2SWAP', 'OP_IFDUP', 'OP_DEPTH', + 'OP_TOALTSTACK', 'OP_FROMALTSTACK', +]); + +const ARITHMETIC_OPS = new Set([ + 'OP_ADD', 'OP_SUB', 'OP_MUL', 'OP_DIV', 'OP_MOD', 'OP_1ADD', 'OP_1SUB', + 'OP_2MUL', 'OP_2DIV', 'OP_NEGATE', 'OP_ABS', 'OP_NOT', 'OP_0NOTEQUAL', + 'OP_BOOLAND', 'OP_BOOLOR', 'OP_NUMEQUAL', 'OP_NUMEQUALVERIFY', + 'OP_NUMNOTEQUAL', 'OP_LESSTHAN', 'OP_GREATERTHAN', 'OP_LESSTHANOREQUAL', + 'OP_GREATERTHANOREQUAL', 'OP_MIN', 'OP_MAX', 'OP_WITHIN', + 'OP_AND', 'OP_OR', 'OP_XOR', 'OP_INVERT', 'OP_LSHIFT', 'OP_RSHIFT', + 'OP_LSHIFTNUM', 'OP_RSHIFTNUM', +]); + +const BYTES_OPS = new Set([ + 'OP_CAT', 'OP_SPLIT', 'OP_SIZE', 'OP_NUM2BIN', 'OP_BIN2NUM', + 'OP_SUBSTR', 'OP_LEFT', 'OP_RIGHT', 'OP_EQUAL', 'OP_EQUALVERIFY', +]); + +const CRYPTO_OPS = new Set([ + 'OP_RIPEMD160', 'OP_SHA1', 'OP_SHA256', 'OP_HASH160', 'OP_HASH256', + 'OP_CHECKSIG', 'OP_CHECKSIGVERIFY', 'OP_CHECKMULTISIG', + 'OP_CHECKMULTISIGVERIFY', 'OP_CODESEPARATOR', +]); + +const CONTROL_OPS = new Set([ + 'OP_IF', 'OP_NOTIF', 'OP_ELSE', 'OP_ENDIF', 'OP_VERIFY', 'OP_RETURN', + 'OP_NOP', 'OP_CHECKLOCKTIMEVERIFY', 'OP_CHECKSEQUENCEVERIFY', +]); + +function categoryOfOpcode(name: string): ByteCategory { + if (SHUFFLE_OPS.has(name)) return 'stack-shuffle'; + if (ARITHMETIC_OPS.has(name)) return 'arithmetic'; + if (BYTES_OPS.has(name)) return 'bytes'; + if (CRYPTO_OPS.has(name)) return 'crypto'; + if (CONTROL_OPS.has(name)) return 'control'; + return 'other'; +} + +// --------------------------------------------------------------------------- +// Serialized-script analysis +// --------------------------------------------------------------------------- + +export interface ConstantUse { + /** Hex of the pushed data (without the length prefix). */ + hex: string; + /** How many times this exact payload is pushed. */ + count: number; + /** Total serialized bytes spent pushing it (payload + prefix, times count). */ + bytes: number; +} + +export interface ScriptMetrics { + scriptBytes: number; + /** Opcodes plus pushes, each counted once. */ + opcodeCount: number; + pushCount: number; + categories: Record; + /** Mnemonic -> occurrence count. Data pushes are keyed as `PUSH`. */ + opcodes: Record; + /** Repeated data payloads, largest total byte cost first. */ + constants: ConstantUse[]; +} + +/** + * Bucket every byte of a serialized script. + * + * Throws on a malformed / truncated push instead of silently dropping the + * tail — a size report that quietly loses bytes is worse than no report. + */ +export function analyzeScriptHex(scriptHex: string): ScriptMetrics { + const hex = scriptHex.trim(); + if (hex.length % 2 !== 0) { + throw new Error(`analyzeScriptHex: odd-length hex (${hex.length} chars)`); + } + const bytes = Buffer.from(hex, 'hex'); + const n = bytes.length; + + const categories = Object.fromEntries(CATEGORIES.map(c => [c, 0])) as Record; + const opcodes: Record = {}; + const constants = new Map(); + + let opcodeCount = 0; + let pushCount = 0; + + /** Bytes + category of the immediately preceding op, for the PICK/ROLL rule. */ + let prevPush: { size: number; category: ByteCategory; dataHex: string | null } | null = null; + + const bump = (name: string) => { opcodes[name] = (opcodes[name] ?? 0) + 1; }; + + let i = 0; + while (i < n) { + const op = bytes[i]!; + + // --- direct pushes ----------------------------------------------------- + if (op >= 0x01 && op <= 0x4b) { + const len = op; + if (i + 1 + len > n) { + throw new Error(`analyzeScriptHex: truncated push at offset ${i} (want ${len} bytes, ${n - i - 1} left)`); + } + const dataHex = bytes.subarray(i + 1, i + 1 + len).toString('hex'); + const size = 1 + len; + categories['const-push'] += size; + bump('PUSH'); + pushCount++; opcodeCount++; + prevPush = { size, category: 'const-push', dataHex }; + i += size; + continue; + } + + if (op === 0x4c || op === 0x4d || op === 0x4e) { + const hdr = op === 0x4c ? 2 : op === 0x4d ? 3 : 5; + if (i + hdr > n) { + throw new Error(`analyzeScriptHex: truncated PUSHDATA header at offset ${i}`); + } + const len = op === 0x4c + ? bytes[i + 1]! + : op === 0x4d + ? bytes.readUInt16LE(i + 1) + : bytes.readUInt32LE(i + 1); + if (i + hdr + len > n) { + throw new Error(`analyzeScriptHex: truncated PUSHDATA body at offset ${i} (want ${len} bytes)`); + } + const dataHex = bytes.subarray(i + hdr, i + hdr + len).toString('hex'); + const size = hdr + len; + categories['const-push'] += size; + bump('PUSH'); + pushCount++; opcodeCount++; + prevPush = { size, category: 'const-push', dataHex }; + i += size; + continue; + } + + // --- single-byte constant pushes -------------------------------------- + if (op === 0x00 || op === 0x4f || (op >= 0x51 && op <= 0x60)) { + categories['small-int-push'] += 1; + bump(OPCODE_NAMES.get(op) ?? `OP_UNKNOWN_${op.toString(16)}`); + pushCount++; opcodeCount++; + prevPush = { size: 1, category: 'small-int-push', dataHex: null }; + i += 1; + continue; + } + + // --- opcodes ----------------------------------------------------------- + const name = OPCODE_NAMES.get(op) ?? `OP_UNKNOWN_${op.toString(16).padStart(2, '0')}`; + const category = categoryOfOpcode(name); + categories[category] += 1; + bump(name); + opcodeCount++; + + // A depth push consumed by PICK/ROLL is stack-access cost, not a constant. + if ((name === 'OP_PICK' || name === 'OP_ROLL') && prevPush) { + categories[prevPush.category] -= prevPush.size; + categories['stack-shuffle'] += prevPush.size; + if (prevPush.dataHex !== null) { + // It was recorded as a data push; un-record it from the constants tally. + const existing = constants.get(prevPush.dataHex); + if (existing) { + existing.count -= 1; + existing.bytes -= prevPush.size; + if (existing.count === 0) constants.delete(prevPush.dataHex); + } + } + } else if (prevPush && prevPush.dataHex !== null) { + // Only now is the previous data push confirmed to be a real constant. + const entry = constants.get(prevPush.dataHex) ?? { count: 0, bytes: 0 }; + entry.count += 1; + entry.bytes += prevPush.size; + constants.set(prevPush.dataHex, entry); + } + + prevPush = null; + i += 1; + } + + // A data push in final position was never confirmed by the loop above. + if (prevPush && prevPush.dataHex !== null) { + const entry = constants.get(prevPush.dataHex) ?? { count: 0, bytes: 0 }; + entry.count += 1; + entry.bytes += prevPush.size; + constants.set(prevPush.dataHex, entry); + } + + const constantList: ConstantUse[] = [...constants.entries()] + .map(([h, v]) => ({ hex: h, count: v.count, bytes: v.bytes })) + .sort((a, b) => b.bytes - a.bytes); + + return { + scriptBytes: n, + opcodeCount, + pushCount, + categories, + opcodes, + constants: constantList, + }; +} + +// --------------------------------------------------------------------------- +// Stack IR analysis +// --------------------------------------------------------------------------- + +export interface StackOpMetrics { + scriptBytes: number; + /** Every op, recursing into `if` arms. An `if` counts as one plus its arms. */ + opCount: number; + /** Ops that only move data around (dup/drop/pick/roll/swap/…). */ + shuffleOps: number; + /** Mnemonic -> count. Structural ops are keyed by the opcode they emit. */ + opcodes: Record; + maxStackDepth?: number; +} + +const STACK_OP_MNEMONIC: Partial> = { + dup: 'OP_DUP', swap: 'OP_SWAP', roll: 'OP_ROLL', pick: 'OP_PICK', + drop: 'OP_DROP', nip: 'OP_NIP', over: 'OP_OVER', rot: 'OP_ROT', + tuck: 'OP_TUCK', if: 'OP_IF', push: 'PUSH', + placeholder: 'PLACEHOLDER', push_codesep_index: 'CODESEP_INDEX', + raw_bytes: 'RAW_BYTES', +}; + +/** Metrics for a Stack IR op sequence, before emission. */ +export function stackOpMetrics(ops: StackOp[], maxStackDepth?: number): StackOpMetrics { + const opcodes: Record = {}; + let opCount = 0; + let shuffleOps = 0; + + const walk = (list: StackOp[]): void => { + for (const op of list) { + opCount++; + const name = op.op === 'opcode' ? op.code : STACK_OP_MNEMONIC[op.op]!; + opcodes[name] = (opcodes[name] ?? 0) + 1; + if (SHUFFLE_OPS.has(name)) shuffleOps++; + if (op.op === 'if') { + walk(op.then); + if (op.else) walk(op.else); + } + } + }; + walk(ops); + + return { + scriptBytes: estimateScriptBytes(ops), + opCount, + shuffleOps, + opcodes, + maxStackDepth, + }; +} From 319d6b0098e7e62bf84be32345b7f243d5d4b677 Mon Sep 17 00:00:00 2001 From: Siggi Date: Fri, 28 Aug 2026 18:15:24 +0200 Subject: [PATCH 2/4] feat(codegen): experimental EC constant pool and liveness stack scheduler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both are opt-in and inert by default, so the 72 goldens, conformance/script-size-baseline.json and cross-tier hex parity are untouched. All 72 fixtures still reproduce their expected-script.hex byte-for-byte, and the Go and Rust cross-compiler golden tests still pass. --ec-constant-pool `fieldMod` pushes the curve's field prime inline at every modular reduction: 20,025 pushes of a 34-byte literal in p256-wallet, 680,850 of its 958,792 bytes. ECTracker gains a pooled slot per constant, and `pushConst` compares the emitted cost of picking that slot against re-pushing the literal and takes the cheaper — so no individual call site can grow. Parameterized by CurveParams / GroupParams, so secp256k1, P-256 and P-384 share one code path. p256-wallet 958,792 -> 304,463 (-68.2%) p384-wallet 1,963,300 -> 463,435 (-76.4%) corpus 13,526,563 -> 6,285,154 (-53.5%), 9 fixtures, none grown Proved equivalent on the real @bsv/sdk engine against OpenSSL signatures on both curves, plus every SEC1 rejection case (r=0, s=0, r=n, s=n, all-zero signature, wrong message, wrong parity, truncated and oversized inputs). --stack-scheduler=liveness Parks a result on the alt stack when the next binding does not consume it, so the operands an ANF chain reads repeatedly stay at depth 0/1; the whole spill group is restored in production order, which is the order an accumulation reads it. Plus commutative operand ordering. Ordering is scored by running the candidate op sequences through the real peephole rather than a byte formula: two consumed operands at depths 1 and 0 emit OP_SWAP OP_SWAP, which `swap-swap` deletes outright — free — while the cheaper-looking reversed order emits one real OP_SWAP and costs a byte. That correction took `arithmetic` from 24 bytes to 18. arithmetic 28 -> 18 B (-35.7%), the hand-derived optimum bounded-loop 42 -> 37 B (-11.9%) ~30 mid-size fixtures -0.1%; 34 changed, none grown Selection is per method: both schedules are lowered and the cheaper one, measured after peephole, is kept — so "the scheduler never grows a method" is structural rather than a property of the greedy heuristic. Spilling is refused in any scope that still has control flow ahead of it. Restoring immediately before an `if` leaves the parent stack in a shape lowerIf's arm reconciliation, declared-result trim and Layer B/C depth invariants were not written for, and it MISCOMPILED if-without-else-multi-temp into accepting a witness the shipping compiler rejects. Byte counts, the other goldens and 4,099 compiler unit tests all passed while that was true; the conformance/witnesses corpus replayed through runDifferentialExecution caught it, 2 of 86 cases. Also: - `conformance/runner/script-metrics.ts` — "where did the bytes go?", the companion to script-size-check.ts's "did anything grow?". Compares named compiler variants and never silently drops a fixture from a size report. - `runDifferentialExecution` accepts both flags, so any experiment can be run through the same source-vs-script oracle. - `--stack-scheduler` rejects an unknown mode instead of falling back to the default: a benchmark that quietly measured the shipping compiler while reporting an experiment is worse than a crash. --- conformance/package.json | 1 + .../runner/__tests__/script-metrics.test.ts | 161 +++++++ conformance/runner/script-metrics.ts | 317 ++++++++++++++ .../src/__tests__/experimental-flags.test.ts | 131 ++++++ packages/runar-cli/src/bin.ts | 2 + packages/runar-cli/src/commands/compile.ts | 27 +- .../src/__tests__/ec-constant-pool.test.ts | 146 +++++++ .../src/__tests__/liveness-scheduler.test.ts | 111 +++++ packages/runar-compiler/src/index.ts | 49 ++- .../src/passes/05-stack-lower.ts | 397 ++++++++++++++++-- .../runar-compiler/src/passes/ec-codegen.ts | 130 +++++- .../src/passes/p256-p384-codegen.ts | 93 ++-- .../ec-constant-pool-equivalence.test.ts | 251 +++++++++++ .../liveness-scheduler-equivalence.test.ts | 144 +++++++ .../src/__tests__/scheduler-headroom.test.ts | 127 ++++++ .../src/oracle/differential-execution.ts | 10 + 16 files changed, 2013 insertions(+), 84 deletions(-) create mode 100644 conformance/runner/__tests__/script-metrics.test.ts create mode 100644 conformance/runner/script-metrics.ts create mode 100644 packages/runar-cli/src/__tests__/experimental-flags.test.ts create mode 100644 packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts create mode 100644 packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts create mode 100644 packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts create mode 100644 packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts create mode 100644 packages/runar-testing/src/__tests__/scheduler-headroom.test.ts diff --git a/conformance/package.json b/conformance/package.json index 0a44c965..9b96f389 100644 --- a/conformance/package.json +++ b/conformance/package.json @@ -12,6 +12,7 @@ "test:parser-only": "tsx runner/index.ts --parser-only", "update-golden": "tsx runner/index.ts --update-golden", "script-size-check": "tsx runner/script-size-check.ts", + "script-metrics": "tsx runner/script-metrics.ts", "fuzz": "tsx fuzzer/index.ts", "fuzz:quick": "tsx fuzzer/index.ts --ir --compilers ts,go,rust,python,zig,ruby,java --render native --num 20", "fuzz:property": "tsx fuzzer/index.ts --property", diff --git a/conformance/runner/__tests__/script-metrics.test.ts b/conformance/runner/__tests__/script-metrics.test.ts new file mode 100644 index 00000000..ff954417 --- /dev/null +++ b/conformance/runner/__tests__/script-metrics.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + parseArgs, + formatTable, + formatComparison, + formatDetail, + measureGolden, + tsSourcePath, + VARIANTS, + type FixtureMetrics, +} from '../script-metrics.js'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function fakeMetrics(fixture: string, scriptBytes: number): FixtureMetrics { + return { + fixture, + source: 'compiled', + scriptBytes, + opcodeCount: scriptBytes, + pushCount: 0, + categories: { + 'const-push': scriptBytes, 'small-int-push': 0, 'stack-shuffle': 0, + arithmetic: 0, bytes: 0, crypto: 0, control: 0, other: 0, + }, + opcodes: { PUSH: 1 }, + constants: [], + }; +} + +// --------------------------------------------------------------------------- + +describe('parseArgs', () => { + it('defaults to reading goldens with a summary table', () => { + const a = parseArgs([]); + expect(a.compileMode).toBe(false); + expect(a.detail).toBe(false); + expect(a.compare).toEqual([]); + }); + + it('--compare implies --compile', () => { + const a = parseArgs(['--compare', 'current,current']); + expect(a.compare).toEqual(['current', 'current']); + expect(a.compileMode).toBe(true); + }); + + it('rejects an unknown argument instead of ignoring it', () => { + // A silently-ignored flag in a benchmark harness reads as "I measured + // that" when nothing was measured. + expect(() => parseArgs(['--nope'])).toThrow(/unknown argument/); + }); +}); + +describe('variants', () => { + it('always offers the shipping default as the comparison base', () => { + expect(VARIANTS.current).toBeDefined(); + expect(VARIANTS.current).toEqual({}); + }); +}); + +describe('tsSourcePath', () => { + it('resolves a fixture that ships a TypeScript source', () => { + const p = tsSourcePath('p2pkh') ?? tsSourcePath('basic-p2pkh'); + expect(p).toMatch(/\.runar\.ts$/); + }); + + it('returns null for a fixture that declares no .runar.ts rather than throwing', () => { + // A size report must skip such a fixture visibly, not crash on it. + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + mkdirSync(join(dir, 'go-only')); + writeFileSync( + join(dir, 'go-only', 'source.json'), + JSON.stringify({ sources: { '.runar.go': './X.runar.go' }, compilers: ['go'] }), + ); + expect(tsSourcePath('go-only', dir)).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('returns null when the fixture has no source.json at all', () => { + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + mkdirSync(join(dir, 'bare')); + expect(tsSourcePath('bare', dir)).toBeNull(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it('throws when source.json points at a file that is not on disk', () => { + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + mkdirSync(join(dir, 'ghost')); + writeFileSync( + join(dir, 'ghost', 'source.json'), + JSON.stringify({ sources: { '.runar.ts': './nope.runar.ts' } }), + ); + expect(() => tsSourcePath('ghost', dir)).toThrow(/missing file/); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('measureGolden', () => { + it('reports the byte length and category split of a golden', () => { + const dir = mkdtempSync(join(tmpdir(), 'runar-metrics-')); + try { + const hexPath = join(dir, 'expected-script.hex'); + // OP_DUP OP_HASH160 <20 bytes> OP_EQUALVERIFY OP_CHECKSIG + writeFileSync(hexPath, `76a914${'ab'.repeat(20)}88ac\n`); + const m = measureGolden('p2pkh-ish', hexPath); + expect(m.scriptBytes).toBe(25); + expect(m.source).toBe('golden'); + expect(m.categories['const-push']).toBe(21); + expect(m.opcodes['OP_CHECKSIG']).toBe(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe('formatting', () => { + it('orders the summary table largest-first', () => { + const table = formatTable([fakeMetrics('small', 10), fakeMetrics('big', 100)]); + expect(table.indexOf('| big |')).toBeLessThan(table.indexOf('| small |')); + }); + + it('reports the delta of the last variant against the first', () => { + const byVariant = new Map([ + ['current', new Map([['fx', fakeMetrics('fx', 1000)]])], + ['tuned', new Map([['fx', fakeMetrics('fx', 250)]])], + ]); + const out = formatComparison(['current', 'tuned'], byVariant); + expect(out).toContain('-75.0%'); + }); + + it('renders a dash for a variant that produced no result', () => { + const byVariant = new Map([ + ['current', new Map([['fx', fakeMetrics('fx', 1000)]])], + ['tuned', new Map()], + ]); + expect(formatComparison(['current', 'tuned'], byVariant)).toContain('| — |'); + }); + + it('lists dominant constants in the detail view', () => { + const m = fakeMetrics('fx', 340); + m.constants = [{ hex: 'ff'.repeat(33), count: 10, bytes: 340 }]; + const out = formatDetail(m); + expect(out).toContain('33 B'); + expect(out).toContain('100.0%'); + }); +}); diff --git a/conformance/runner/script-metrics.ts b/conformance/runner/script-metrics.ts new file mode 100644 index 00000000..76acb6b8 --- /dev/null +++ b/conformance/runner/script-metrics.ts @@ -0,0 +1,317 @@ +/** + * Script-size instrumentation runner. + * + * `script-size-check.ts` answers "did any fixture grow?"; this answers the + * next question: "where did the bytes go?". For every conformance fixture it + * buckets the serialized script by byte category and reports the constants + * that dominate, so an optimization can be aimed at the term that actually + * costs something rather than at whatever is easiest to change. + * + * Two sources of script bytes: + * + * default — read the checked-in `expected-script.hex`. Fast, needs no + * compilation, and is exactly what CI ships. + * --compile — recompile each fixture's `.runar.ts` through the TS reference + * compiler with a given option set. This is how an experimental + * flag is benchmarked against the baseline; pass `--compare` to + * run several option sets and print the deltas. + * + * Usage: + * tsx runner/script-metrics.ts # goldens, markdown table + * tsx runner/script-metrics.ts --json out.json # machine-readable + * tsx runner/script-metrics.ts --fixture p256-wallet --detail + * tsx runner/script-metrics.ts --compile --compare current,liveness + * + * Read-only: it never writes a golden or a baseline. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { + analyzeScriptHex, + type ByteCategory, + type ScriptMetrics, +} from '../../packages/runar-compiler/src/metrics/script-metrics.js'; +import { compile, type CompileOptions } from '../../packages/runar-compiler/src/index.js'; +import { CONFORMANCE_ROOT, TESTS_DIR, discoverFixtures } from './script-size-check.js'; + +const REPO_ROOT = resolve(CONFORMANCE_ROOT, '..'); + +// --------------------------------------------------------------------------- +// Fixture sources +// --------------------------------------------------------------------------- + +interface SourceConfig { + sources?: Record; + compilers?: string[]; +} + +/** Absolute path to a fixture's `.runar.ts` source, or null if it ships none. */ +export function tsSourcePath(fixture: string, testsDir: string = TESTS_DIR): string | null { + const configFile = join(testsDir, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf8')) as SourceConfig; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(testsDir, fixture, rel); + if (!existsSync(abs)) { + throw new Error(`source.json for '${fixture}' points at a missing file: ${abs}`); + } + return abs; +} + +// --------------------------------------------------------------------------- +// Option sets ("variants") a benchmark can compare +// --------------------------------------------------------------------------- + +/** + * Named compiler configurations. `current` is the shipping default; every + * other entry is an experimental flag combination. Adding a variant here is + * all it takes to get it into the comparison table. + */ +export const VARIANTS: Record = { + current: {}, + 'ec-pool': { ecConstantPool: true }, + liveness: { schedulerMode: 'liveness' }, + both: { ecConstantPool: true, schedulerMode: 'liveness' }, +}; + +// --------------------------------------------------------------------------- +// Measurement +// --------------------------------------------------------------------------- + +export interface FixtureMetrics extends ScriptMetrics { + fixture: string; + /** Where the bytes came from: the checked-in golden or a live compile. */ + source: 'golden' | 'compiled'; +} + +export function measureGolden(fixture: string, hexPath: string): FixtureMetrics { + const hex = readFileSync(hexPath, 'utf8').replace(/\s+/g, ''); + return { fixture, source: 'golden', ...analyzeScriptHex(hex) }; +} + +export function measureCompiled( + fixture: string, + sourcePath: string, + options: CompileOptions, +): FixtureMetrics { + const source = readFileSync(sourcePath, 'utf8'); + const result = compile(source, { ...options, fileName: sourcePath }); + if (!result.success || result.scriptHex === undefined) { + const errs = result.diagnostics.filter(d => d.severity === 'error').map(d => d.message); + throw new Error(`compile failed for '${fixture}': ${errs.join('; ') || 'no scriptHex'}`); + } + return { fixture, source: 'compiled', ...analyzeScriptHex(result.scriptHex) }; +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +const REPORT_CATEGORIES: ByteCategory[] = [ + 'const-push', 'stack-shuffle', 'arithmetic', 'bytes', 'crypto', + 'small-int-push', 'control', 'other', +]; + +function pct(part: number, whole: number): string { + if (whole === 0) return '0.0%'; + return `${((100 * part) / whole).toFixed(1)}%`; +} + +export function formatTable(rows: FixtureMetrics[]): string { + const sorted = [...rows].sort((a, b) => b.scriptBytes - a.scriptBytes); + const head = ['fixture', 'bytes', 'ops', ...REPORT_CATEGORIES]; + const lines = [ + `| ${head.join(' | ')} |`, + `|${head.map((_, i) => (i === 0 ? '---' : '---:')).join('|')}|`, + ]; + for (const r of sorted) { + const cells = REPORT_CATEGORIES.map(c => { + const v = r.categories[c]; + return v === 0 ? '—' : `${v} (${pct(v, r.scriptBytes)})`; + }); + lines.push(`| ${r.fixture} | ${r.scriptBytes} | ${r.opcodeCount} | ${cells.join(' | ')} |`); + } + return lines.join('\n'); +} + +/** Per-fixture detail: the constants that dominate, and the opcode histogram. */ +export function formatDetail(m: FixtureMetrics, topN = 8): string { + const out: string[] = []; + out.push(`### ${m.fixture} — ${m.scriptBytes} bytes, ${m.opcodeCount} ops (${m.source})`); + out.push(''); + out.push('| category | bytes | share |'); + out.push('|---|---:|---:|'); + for (const c of REPORT_CATEGORIES) { + const v = m.categories[c]; + if (v > 0) out.push(`| ${c} | ${v} | ${pct(v, m.scriptBytes)} |`); + } + out.push(''); + if (m.constants.length > 0) { + out.push('| repeated constant | size | pushes | total bytes | share |'); + out.push('|---|---:|---:|---:|---:|'); + for (const c of m.constants.slice(0, topN)) { + const label = c.hex.length > 24 ? `${c.hex.slice(0, 20)}…` : c.hex; + out.push(`| \`${label}\` | ${c.hex.length / 2} B | ${c.count} | ${c.bytes} | ${pct(c.bytes, m.scriptBytes)} |`); + } + out.push(''); + } + const ops = Object.entries(m.opcodes).sort((a, b) => b[1] - a[1]).slice(0, topN); + out.push(`Top opcodes: ${ops.map(([k, v]) => `${k}×${v}`).join(', ')}`); + return out.join('\n'); +} + +/** Side-by-side variant comparison for one fixture set. */ +export function formatComparison( + variantNames: string[], + byVariant: Map>, +): string { + const base = variantNames[0]!; + const fixtures = [...(byVariant.get(base)?.keys() ?? [])] + .sort((a, b) => (byVariant.get(base)!.get(b)!.scriptBytes) - (byVariant.get(base)!.get(a)!.scriptBytes)); + const head = ['fixture', ...variantNames.map(v => (v === base ? `${v} (base)` : `${v}`)), 'delta']; + const lines = [ + `| ${head.join(' | ')} |`, + `|${head.map((_, i) => (i === 0 ? '---' : '---:')).join('|')}|`, + ]; + for (const fx of fixtures) { + const baseBytes = byVariant.get(base)!.get(fx)!.scriptBytes; + const cells = variantNames.map(v => { + const m = byVariant.get(v)?.get(fx); + return m ? String(m.scriptBytes) : '—'; + }); + const last = byVariant.get(variantNames[variantNames.length - 1]!)?.get(fx); + const delta = last && baseBytes > 0 + ? `${(((last.scriptBytes - baseBytes) / baseBytes) * 100).toFixed(1)}%` + : '—'; + lines.push(`| ${fx} | ${cells.join(' | ')} | ${delta} |`); + } + return lines.join('\n'); +} + +// --------------------------------------------------------------------------- +// CLI +// --------------------------------------------------------------------------- + +interface Args { + json?: string; + fixture?: string; + detail: boolean; + compileMode: boolean; + compare: string[]; + top: number; +} + +export function parseArgs(argv: string[]): Args { + const args: Args = { detail: false, compileMode: false, compare: [], top: 8 }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]!; + if (a === '--json') args.json = argv[++i]; + else if (a === '--fixture') args.fixture = argv[++i]; + else if (a === '--detail') args.detail = true; + else if (a === '--compile') args.compileMode = true; + else if (a === '--compare') { args.compare = (argv[++i] ?? '').split(',').filter(Boolean); args.compileMode = true; } + else if (a === '--top') args.top = Number(argv[++i]); + else if (a === '--help' || a === '-h') { args.detail = false; printHelp(); process.exit(0); } + else throw new Error(`script-metrics: unknown argument '${a}'`); + } + return args; +} + +function printHelp(): void { + console.log(` +script-metrics — where do a fixture's script bytes go? + + --fixture measure one fixture instead of all + --detail per-fixture category / constant / opcode breakdown + --compile recompile from .runar.ts instead of reading the golden + --compare a,b,c compile under several named variants and diff (implies --compile) + --json write machine-readable results + --top rows in the constant / opcode lists (default 8) + +Variants available: ${Object.keys(VARIANTS).join(', ')} +`.trim()); +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const fixtures = discoverFixtures(); + const selected = args.fixture + ? new Map(fixtures.has(args.fixture) ? [[args.fixture, fixtures.get(args.fixture)!]] : []) + : fixtures; + if (selected.size === 0) { + throw new Error(`no fixtures selected${args.fixture ? ` (unknown fixture '${args.fixture}')` : ''}`); + } + + if (args.compare.length > 0) { + for (const name of args.compare) { + if (!(name in VARIANTS)) { + throw new Error(`unknown variant '${name}'. Known: ${Object.keys(VARIANTS).join(', ')}`); + } + } + const byVariant = new Map>(); + const skipped: string[] = []; + for (const name of args.compare) { + const perFixture = new Map(); + for (const [fx] of selected) { + const src = tsSourcePath(fx); + if (src === null) { if (name === args.compare[0]) skipped.push(fx); continue; } + perFixture.set(fx, measureCompiled(fx, src, VARIANTS[name]!)); + } + byVariant.set(name, perFixture); + } + console.log(formatComparison(args.compare, byVariant)); + if (skipped.length > 0) { + // Never silently drop a fixture from a size report. + console.log(`\n_Skipped (no .runar.ts source): ${skipped.join(', ')}_`); + } + if (args.json) { + const payload = Object.fromEntries( + [...byVariant].map(([v, m]) => [v, Object.fromEntries(m)]), + ); + writeFileSync(args.json, `${JSON.stringify(payload, null, 2)}\n`); + } + return; + } + + const rows: FixtureMetrics[] = []; + const skipped: string[] = []; + for (const [fx, hexPath] of selected) { + if (args.compileMode) { + const src = tsSourcePath(fx); + if (src === null) { skipped.push(fx); continue; } + rows.push(measureCompiled(fx, src, VARIANTS.current!)); + } else { + rows.push(measureGolden(fx, hexPath)); + } + } + + if (args.detail) { + for (const r of [...rows].sort((a, b) => b.scriptBytes - a.scriptBytes)) { + console.log(formatDetail(r, args.top)); + console.log(''); + } + } else { + console.log(formatTable(rows)); + } + if (skipped.length > 0) { + console.log(`\n_Skipped (no .runar.ts source): ${skipped.join(', ')}_`); + } + if (args.json) { + writeFileSync(args.json, `${JSON.stringify(rows, null, 2)}\n`); + } +} + +const isMain = process.argv[1] !== undefined + && resolve(process.argv[1]).endsWith(join('runner', 'script-metrics.ts')); +if (isMain) { + try { + main(); + } catch (err) { + console.error(`script-metrics: ${(err as Error).message}`); + process.exit(1); + } +} + +export { REPO_ROOT }; diff --git a/packages/runar-cli/src/__tests__/experimental-flags.test.ts b/packages/runar-cli/src/__tests__/experimental-flags.test.ts new file mode 100644 index 00000000..2626d233 --- /dev/null +++ b/packages/runar-cli/src/__tests__/experimental-flags.test.ts @@ -0,0 +1,131 @@ +// --------------------------------------------------------------------------- +// Tests for the experimental size-optimizer flags on the TS CLI: +// --stack-scheduler +// --ec-constant-pool +// +// Both change emitted bytes when enabled and MUST be inert when absent, since +// the checked-in goldens, `conformance/script-size-baseline.json` and the +// cross-tier hex parity gate all assume the default path. +// +// See docs/experiments/script-size-optimizer-results.md. +// --------------------------------------------------------------------------- + +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as os from 'node:os'; + +/** The smallest contract whose bytes come entirely from the generic scheduler. */ +const ARITHMETIC = ` +import { SmartContract, assert } from 'runar-lang'; + +class Arithmetic extends SmartContract { + readonly target: bigint; + constructor(target: bigint) { + super(target); + this.target = target; + } + public verify(a: bigint, b: bigint): void { + const sum: bigint = a + b; + const diff: bigint = a - b; + const prod: bigint = a * b; + const quot: bigint = a / b; + const result: bigint = sum + diff + prod + quot; + assert(result === this.target); + } +} +`; + +let workDir: string; + +beforeAll(() => { + workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'runar-cli-exp-')); +}); + +afterAll(() => { + if (workDir) fs.rmSync(workDir, { recursive: true, force: true }); +}); + +describe('experimental size-optimizer flags', () => { + let compileCommand: typeof import('../commands/compile.js').compileCommand; + + beforeAll(async () => { + const sourceEntry = path.resolve(process.cwd(), 'packages/runar-compiler/src/index.ts'); + if (fs.existsSync(sourceEntry)) { + const { pathToFileURL } = await import('node:url'); + await import(pathToFileURL(sourceEntry).href); + } else { + await import('runar-compiler'); + } + }, 60_000); + + beforeEach(async () => { + ({ compileCommand } = await import('../commands/compile.js')); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + /** Compile ARITHMETIC with the given options and return the printed hex. */ + async function hexWith(options: Record, tag: string): Promise { + const srcPath = path.join(workDir, `Arithmetic-${tag}.runar.ts`); + fs.writeFileSync(srcPath, ARITHMETIC); + const outDir = path.join(workDir, `out-${tag}`); + fs.mkdirSync(outDir, { recursive: true }); + + const writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + const originalExitCode = process.exitCode; + + await compileCommand([srcPath], { output: outDir, hex: true, ...options }); + + const printed = writeSpy.mock.calls.map(c => String(c[0])).join('').trim(); + process.exitCode = originalExitCode; + writeSpy.mockRestore(); + return printed; + } + + it('is inert when no experimental flag is passed', async () => { + const bare = await hexWith({}, 'bare'); + const explicit = await hexWith({ stackScheduler: 'current' }, 'explicit'); + expect(bare).toBe(explicit); + // The shipping schedule for this contract; also the checked-in golden for + // conformance/tests/arithmetic modulo the constructor placeholder. + expect(bare).toBe('6e9352795279945379537995547a547a96537a537a937b937c93009c'); + expect(bare.length / 2).toBe(28); + }); + + it('--stack-scheduler liveness reschedules and shrinks', async () => { + const out = await hexWith({ stackScheduler: 'liveness' }, 'liveness'); + expect(out).toBe('6e936b6e946b6e956b966c6c6c939393009c'); + expect(out.length / 2).toBe(18); + }); + + it('rejects an unknown scheduler mode instead of silently using the default', async () => { + // A benchmark run that quietly measured the shipping compiler while + // reporting an experiment is worse than a crash. + const errSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + const originalExitCode = process.exitCode; + + const srcPath = path.join(workDir, 'Arithmetic-bad.runar.ts'); + fs.writeFileSync(srcPath, ARITHMETIC); + const outDir = path.join(workDir, 'out-bad'); + fs.mkdirSync(outDir, { recursive: true }); + + await compileCommand([srcPath], { output: outDir, hex: true, stackScheduler: 'bogus' }); + + const errors = errSpy.mock.calls.map(c => String(c[0])).join('\n'); + expect(errors).toMatch(/unknown mode 'bogus'/); + process.exitCode = originalExitCode; + }); + + it('--ec-constant-pool is inert on a contract with no EC operations', async () => { + const bare = await hexWith({}, 'nopool'); + const pooled = await hexWith({ ecConstantPool: true }, 'pool'); + expect(pooled).toBe(bare); + }); +}); diff --git a/packages/runar-cli/src/bin.ts b/packages/runar-cli/src/bin.ts index 1a4ade66..506f1a64 100644 --- a/packages/runar-cli/src/bin.ts +++ b/packages/runar-cli/src/bin.ts @@ -43,6 +43,8 @@ program .option('--ir', 'include IR in artifact') .option('--asm', 'print ASM to stdout') .option('--disable-constant-folding', 'disable ANF constant folding pass') + .option('--ec-constant-pool', 'EXPERIMENTAL: pool repeated EC curve constants (changes emitted bytes)') + .option('--stack-scheduler ', 'EXPERIMENTAL: operand scheduling — current|liveness', 'current') .option('--from-ir ', 'compile from an ANF IR JSON file (skips parse/validate/typecheck/anf-lower)') .option('--hex', 'print only the script hex to stdout (no artifact JSON)') .option('--parse-only', 'stop after parse + validate; print "parser ok" on success (requires source input)') diff --git a/packages/runar-cli/src/commands/compile.ts b/packages/runar-cli/src/commands/compile.ts index 74afbd29..97a71051 100644 --- a/packages/runar-cli/src/commands/compile.ts +++ b/packages/runar-cli/src/commands/compile.ts @@ -12,6 +12,8 @@ interface CompileOptions { ir?: boolean; asm?: boolean; disableConstantFolding?: boolean; + ecConstantPool?: boolean; + stackScheduler?: string; fromIr?: string; hex?: boolean; parseOnly?: boolean; @@ -72,6 +74,19 @@ function repoRelativeFileName(absSourcePath: string): string { * 3. Write the resulting artifact JSON to the output directory. * 4. Optionally print the ASM to stdout. */ +/** + * Validate `--stack-scheduler`. An unrecognised mode is an error rather than a + * silent fall back to the default: a benchmark run that quietly measured the + * shipping compiler while reporting an experiment would be worse than a crash. + */ +function schedulerMode(options: { stackScheduler?: string }): 'current' | 'liveness' { + const mode = options.stackScheduler ?? 'current'; + if (mode !== 'current' && mode !== 'liveness') { + throw new Error(`--stack-scheduler: unknown mode '${mode}' (expected current|liveness)`); + } + return mode; +} + export async function compileCommand( files: string[], options: CompileOptions, @@ -81,10 +96,10 @@ export async function compileCommand( // Dynamically import the compiler to avoid hard failures if it's not // yet fully built (the compiler package may still be under development). - type CompileFn = (source: string, options?: { fileName?: string; disableConstantFolding?: boolean; parseOnly?: boolean }) => unknown; + type CompileFn = (source: string, options?: { fileName?: string; disableConstantFolding?: boolean; ecConstantPool?: boolean; schedulerMode?: 'current' | 'liveness'; parseOnly?: boolean }) => unknown; type CompileFromANFFn = ( program: unknown, - options?: { disableConstantFolding?: boolean }, + options?: { disableConstantFolding?: boolean; ecConstantPool?: boolean; schedulerMode?: 'current' | 'liveness' }, ) => { scriptHex: string; scriptAsm: string }; type LoadANFFn = (json: string) => unknown; @@ -174,7 +189,11 @@ export async function compileCommand( let result: { scriptHex: string; scriptAsm: string }; try { - result = compileFromANF(program, { disableConstantFolding: options.disableConstantFolding }); + result = compileFromANF(program, { + disableConstantFolding: options.disableConstantFolding, + ecConstantPool: options.ecConstantPool, + schedulerMode: schedulerMode(options), + }); } catch (err) { console.error(` Compilation error: ${(err as Error).message}`); process.exitCode = 1; @@ -250,6 +269,8 @@ export async function compileCommand( compileResult = compile(source, { fileName: resolvedPath, disableConstantFolding: options.disableConstantFolding, + ecConstantPool: options.ecConstantPool, + schedulerMode: schedulerMode(options), parseOnly: options.parseOnly, }) as CompileResultLike; } catch (err) { diff --git a/packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts b/packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts new file mode 100644 index 00000000..4654ccfb --- /dev/null +++ b/packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts @@ -0,0 +1,146 @@ +/** + * EC constant pooling — size and default-invariance. + * + * `cFieldMod` / `fieldMod` push the curve's field prime inline at EVERY + * modular reduction. On `conformance/tests/p256-wallet` that is 20,025 pushes + * of a 34-byte literal — 680,850 of the fixture's 958,792 bytes, 71 %. The + * prime is a compile-time constant; parking one copy in a stack slot and + * copying it with `push d; OP_PICK` costs 2-3 bytes instead of 34. + * + * This file pins two things: + * 1. with pooling OFF the emitters are byte-identical to what ships today + * (so no golden, baseline, or cross-tier parity gate can move), and + * 2. with pooling ON the scripts actually shrink, by the amount the + * arithmetic predicts rather than by "some". + * + * Semantic equivalence is proved separately, against OpenSSL signatures on the + * real interpreter, in + * `packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts`. + */ + +import { describe, it, expect } from 'vitest'; +import { + emitVerifyECDSA_P256, emitVerifyECDSA_P384, + emitP256Mul, emitP256MulGen, emitP256Add, emitP256OnCurve, + emitP384Mul, emitP384Add, +} from '../passes/p256-p384-codegen.js'; +import { + emitEcAdd, emitEcMul, emitEcMulGen, +} from '../passes/ec-codegen.js'; +import { emitMethod } from '../passes/06-emit.js'; +import { estimateScriptBytes } from '../metrics/cost-model.js'; +import { analyzeScriptHex } from '../metrics/script-metrics.js'; +import { encodeScriptNumber } from '../passes/push-encoding.js'; +import type { StackOp } from '../ir/index.js'; +import type { EcCodegenOptions } from '../passes/ec-codegen.js'; + +type Emitter = (emit: (op: StackOp) => void, opts?: EcCodegenOptions) => void; + +function opsOf(emitter: Emitter, opts?: EcCodegenOptions): StackOp[] { + const ops: StackOp[] = []; + emitter(op => ops.push(op), opts); + return ops; +} + +function hexOf(ops: StackOp[]): string { + return emitMethod({ name: 'probe', ops, maxStackDepth: 0 }).scriptHex; +} + +function bytesOf(emitter: Emitter, opts?: EcCodegenOptions): number { + return estimateScriptBytes(opsOf(emitter, opts)); +} + +/** + * Net stack effect of an op sequence, counting only the ops whose effect is + * unambiguous from the Stack IR alone (pushes and pops). Opcodes are opaque + * here, so this is a same-shape comparison between two variants of the SAME + * emitter, not an absolute depth model — which is all the pool needs to prove: + * every slot it pushes, it releases. + */ +function netStackEffect(ops: StackOp[]): number { + let net = 0; + const walk = (list: StackOp[]): void => { + for (const op of list) { + if (op.op === 'push' || op.op === 'dup' || op.op === 'over' || op.op === 'tuck' + || op.op === 'placeholder' || op.op === 'push_codesep_index') net++; + else if (op.op === 'drop' || op.op === 'nip') net--; + // A pick/roll is always preceded by a `push` of the depth, already + // counted above. OP_PICK consumes that depth and pushes a copy (net 0); + // OP_ROLL consumes it and relocates an existing item (net -1). + else if (op.op === 'roll') net--; + else if (op.op === 'if') { walk(op.then); if (op.else) walk(op.else); } + } + }; + walk(ops); + return net; +} + +/** Every emitter that should benefit, with its shipping byte count. */ +const EMITTERS: Array<[string, Emitter, number]> = [ + ['emitVerifyECDSA_P256', emitVerifyECDSA_P256, 974024], + ['emitVerifyECDSA_P384', emitVerifyECDSA_P384, 1987394], + ['emitP256Mul', emitP256Mul, 459746], + ['emitP256MulGen', emitP256MulGen, 459812], + ['emitP256Add', emitP256Add, 19906], + ['emitP384Mul', emitP384Mul, 927350], + ['emitP384Add', emitP384Add, 46710], + ['emitEcAdd', emitEcAdd, 25426], + ['emitEcMul', emitEcMul, 428676], + ['emitEcMulGen', emitEcMulGen, 428742], +]; + +describe('pooling OFF is the shipping default', () => { + it.each(EMITTERS)('%s emits its documented byte count', (_name, emitter, want) => { + expect(bytesOf(emitter)).toBe(want); + }); + + it.each(EMITTERS)('%s is byte-identical with an explicit constantPool:false', (_n, emitter) => { + expect(hexOf(opsOf(emitter))).toBe(hexOf(opsOf(emitter, { constantPool: false }))); + }); + + it('an empty options object changes nothing', () => { + expect(hexOf(opsOf(emitP256OnCurve))).toBe(hexOf(opsOf(emitP256OnCurve, {}))); + }); +}); + +describe('pooling ON removes the repeated prime pushes', () => { + it('collapses the P-256 field prime from 34 bytes a push to a pick', () => { + const before = analyzeScriptHex(hexOf(opsOf(emitVerifyECDSA_P256))); + const after = analyzeScriptHex(hexOf(opsOf(emitVerifyECDSA_P256, { constantPool: true }))); + + const p = Buffer.from( + encodeScriptNumber(0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn), + ).toString('hex'); + const beforeP = before.constants.find(c => c.hex === p)!; + expect(beforeP.count).toBeGreaterThan(15000); + + const afterP = after.constants.find(c => c.hex === p); + // At most a handful survive: the pool slot itself, plus any site that + // genuinely cannot see the slot. + expect(afterP?.count ?? 0).toBeLessThan(10); + }); + + it.each(EMITTERS)('%s shrinks by more than half', (_name, emitter, before) => { + const after = bytesOf(emitter, { constantPool: true }); + expect(after).toBeLessThan(before * 0.5); + }); + + it('brings verifyECDSA_P256 close to the arithmetic prediction', () => { + // 20,025-ish reductions x ~31 bytes saved each on the p256-wallet fixture; + // the bare emitter carries the same reduction count. Predicted landing + // zone is ~330 kB. Allow slack, but fail if it lands nowhere near. + const after = bytesOf(emitVerifyECDSA_P256, { constantPool: true }); + expect(after).toBeGreaterThan(200_000); + expect(after).toBeLessThan(420_000); + }); + + it('adds at most two resident slots', () => { + // The pool is two extra stack items per tracker (p and n). Real max-depth + // is measured on the interpreter in the equivalence test; here we only pin + // that the emitter still balances — pool slots pushed are pool slots + // released, so the net stack effect is unchanged. + const off = opsOf(emitP256Add); + const on = opsOf(emitP256Add, { constantPool: true }); + expect(netStackEffect(on)).toBe(netStackEffect(off)); + }); +}); diff --git a/packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts b/packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts new file mode 100644 index 00000000..00751b8c --- /dev/null +++ b/packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts @@ -0,0 +1,111 @@ +/** + * Liveness-aware stack scheduling — size and default-invariance. + * + * ANF names every intermediate, and the current lowering pushes each result on + * top of the operands that produced it. In an arithmetic chain that reads the + * same two values repeatedly, every result buries them one slot deeper, so the + * next access costs a `push d; OP_PICK` pair instead of a 1-byte `OP_2DUP`. + * `conformance/tests/arithmetic` spends 16 of its 28 bytes exactly that way. + * + * The `liveness` scheduler parks a result on the alt stack when the next + * binding does not want it, keeping the hot operands at depth 0/1, and + * restores the whole spill group in one go before the first binding that + * needs any of it. Restoring en masse puts the values back in production + * order (first-spilled on top), which is the order an ANF accumulation chain + * consumes them in. + * + * Pinned here: (1) `current` mode is byte-identical to what ships, and + * (2) `liveness` mode actually shrinks the arithmetic-heavy fixtures. + * Semantic equivalence is proved on the real interpreter in + * `packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts`. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'fs'; +import { join, resolve } from 'path'; +import { compile } from '../index.js'; +import { analyzeScriptHex } from '../metrics/script-metrics.js'; + +const CONFORMANCE_DIR = join(__dirname, '..', '..', '..', '..', 'conformance', 'tests'); + +function tsSourceFor(fixture: string): string | null { + const configFile = join(CONFORMANCE_DIR, fixture, 'source.json'); + if (!existsSync(configFile)) return null; + const config = JSON.parse(readFileSync(configFile, 'utf-8')) as { sources?: Record }; + const rel = config.sources?.['.runar.ts']; + if (rel === undefined) return null; + const abs = resolve(CONFORMANCE_DIR, fixture, rel); + return existsSync(abs) ? abs : null; +} + +function hexFor(fixture: string, liveness: boolean): string { + const path = tsSourceFor(fixture)!; + const result = compile(readFileSync(path, 'utf-8'), { + fileName: path, + disableConstantFolding: true, + ...(liveness ? { schedulerMode: 'liveness' as const } : {}), + }); + expect(result.success, result.diagnostics.map(d => d.message).join('; ')).toBe(true); + return result.scriptHex!; +} + +const ALL_FIXTURES = readdirSync(CONFORMANCE_DIR, { withFileTypes: true }) + .filter(e => e.isDirectory()) + .map(e => e.name) + .filter(name => tsSourceFor(name) !== null + && existsSync(join(CONFORMANCE_DIR, name, 'expected-script.hex'))) + .sort(); + +describe('scheduler mode "current" is the shipping default', () => { + it.each(ALL_FIXTURES)('%s matches its golden', (fixture) => { + const golden = readFileSync(join(CONFORMANCE_DIR, fixture, 'expected-script.hex'), 'utf-8') + .replace(/\s+/g, ''); + expect(hexFor(fixture, false)).toBe(golden); + }); +}); + +describe('scheduler mode "liveness"', () => { + it('never grows a fixture', () => { + // A scheduler that trades bytes for bytes is not an optimization. Every + // spill decision goes through the cost model, so growth is a bug. + const grew: string[] = []; + for (const fixture of ALL_FIXTURES) { + const before = hexFor(fixture, false).length / 2; + const after = hexFor(fixture, true).length / 2; + if (after > before) grew.push(`${fixture}: ${before} -> ${after}`); + } + expect(grew).toEqual([]); + }); + + it('shrinks the arithmetic fixture by more than 10 %', () => { + const before = hexFor('arithmetic', false).length / 2; + const after = hexFor('arithmetic', true).length / 2; + expect(before).toBe(28); + expect(1 - after / before).toBeGreaterThan(0.1); + }); + + it('replaces PICK/ROLL traffic with alt-stack round trips on arithmetic', () => { + const before = analyzeScriptHex(hexFor('arithmetic', false)); + const after = analyzeScriptHex(hexFor('arithmetic', true)); + const shuffle = (m: typeof before) => m.categories['stack-shuffle']; + expect(shuffle(after)).toBeLessThan(shuffle(before)); + expect(after.opcodes['OP_PICK'] ?? 0).toBeLessThan(before.opcodes['OP_PICK'] ?? 0); + expect(after.opcodes['OP_TOALTSTACK'] ?? 0).toBeGreaterThan(0); + }); + + it('balances every spill it introduces', () => { + // A static count is NOT a balance proof on its own: `sha256-finalize` + // already emits 896 OP_TOALTSTACK against 897 OP_FROMALTSTACK, because + // one arm of an `if` pushes to the alt stack and the other does not, and + // both arms are counted. So the invariant is a DELTA one: whatever the + // scheduler adds must be added in pairs. + for (const fixture of ALL_FIXTURES) { + const before = analyzeScriptHex(hexFor(fixture, false)); + const after = analyzeScriptHex(hexFor(fixture, true)); + const to = (after.opcodes['OP_TOALTSTACK'] ?? 0) - (before.opcodes['OP_TOALTSTACK'] ?? 0); + const from = (after.opcodes['OP_FROMALTSTACK'] ?? 0) - (before.opcodes['OP_FROMALTSTACK'] ?? 0); + expect(to, `${fixture}: unbalanced spill traffic`).toBe(from); + expect(to, `${fixture}: negative spill count`).toBeGreaterThanOrEqual(0); + } + }); +}); diff --git a/packages/runar-compiler/src/index.ts b/packages/runar-compiler/src/index.ts index fc91507b..298b69c3 100644 --- a/packages/runar-compiler/src/index.ts +++ b/packages/runar-compiler/src/index.ts @@ -67,6 +67,14 @@ export { foldConstants } from './optimizer/constant-fold.js'; export { eliminateDeadBindings } from './optimizer/dce.js'; export { assembleArtifact } from './artifact/assembler.js'; +// Script-byte cost model + size instrumentation. Read-only: nothing here +// changes compilation output, but `estimateScriptBytes` is the metric any +// size-directed optimizer pass must compare candidates with, and it is +// asserted byte-exact against the emitter over the whole conformance corpus. +export { sizeOfStackOp, sizeOfPushValue, estimateScriptBytes } from './metrics/cost-model.js'; +export { analyzeScriptHex, stackOpMetrics } from './metrics/script-metrics.js'; +export type { ByteCategory, ConstantUse, ScriptMetrics, StackOpMetrics } from './metrics/script-metrics.js'; + export type { CompilerDiagnostic, Severity } from './errors.js'; export { CompilerError, ParseError, ValidationError, TypeError, makeDiagnostic } from './errors.js'; @@ -158,6 +166,33 @@ export interface CompileOptions { */ disablePeephole?: boolean; + /** + * EXPERIMENTAL. Park each curve's field prime / group order in a stack slot + * inside the EC codegen modules instead of re-pushing the 33- or 49-byte + * literal at every modular reduction. + * + * Default false, and the emitters take an untouched code path when it is — + * so the checked-in goldens, `conformance/script-size-baseline.json`, and + * cross-tier hex parity are all unaffected while this is off. Turning it on + * changes the emitted bytes and is therefore a TS-tier-only experiment until + * the transformation is ported to the other six compilers. Measured effect: + * `verifyECDSA_P256` 974,024 -> 319,693 bytes (-67 %). + * + * See `docs/experiments/script-size-optimization-baseline.md`. + */ + ecConstantPool?: boolean; + + /** + * EXPERIMENTAL. Operand scheduling strategy for the ANF -> Stack pass. + * + * `'current'` (default) ships today's bytes. `'liveness'` parks a result on + * the alt stack when the next binding does not consume it, so the operands a + * chain reads repeatedly stay at depth 0/1 instead of sinking one slot per + * binding. TS-tier-only experiment while it is opt-in; see + * `docs/experiments/stack-scheduler-design.md`. + */ + schedulerMode?: 'current' | 'liveness'; + /** Called between compilation passes with the current stage name and progress percentage (0-100). */ onProgress?: (stage: string, percent: number) => void; } @@ -463,7 +498,10 @@ export function compile(source: string, options?: CompileOptions): CompileResult // Pass 5-6: Stack lower + Peephole optimize + Emit try { onProgress?.('Stack lowering', 60); - const stackProgram = lowerToStack(optimizedAnf); + const stackProgram = lowerToStack(optimizedAnf, { + ecConstantPool: opts.ecConstantPool === true, + schedulerMode: opts.schedulerMode, + }); // Apply peephole optimization to each method's ops (runs on Stack IR, // after the ANF conformance boundary, so it doesn't affect cross-compiler @@ -552,6 +590,10 @@ export interface CompileFromANFOptions { disableEcOptimizer?: boolean; /** If true, skip the Stack IR peephole optimizer. See CompileOptions for context. */ disablePeephole?: boolean; + /** EXPERIMENTAL. Pool repeated EC curve constants. See CompileOptions. */ + ecConstantPool?: boolean; + /** EXPERIMENTAL. Operand scheduling strategy. See CompileOptions. */ + schedulerMode?: 'current' | 'liveness'; } export interface CompileFromANFResult { @@ -621,7 +663,10 @@ export function compileFromANF( // EC optimizer delegates internally to optimizer/dce.ts for dead-binding cleanup. const optimizedAnf = opts.disableEcOptimizer ? anf : optimizeEC(anf); - const stackProgram = lowerToStack(optimizedAnf); + const stackProgram = lowerToStack(optimizedAnf, { + ecConstantPool: opts.ecConstantPool === true, + schedulerMode: opts.schedulerMode, + }); if (!opts.disablePeephole) { for (const method of stackProgram.methods) { method.ops = optimizeStackIR(method.ops); diff --git a/packages/runar-compiler/src/passes/05-stack-lower.ts b/packages/runar-compiler/src/passes/05-stack-lower.ts index ec9c57ce..4f2f55b1 100644 --- a/packages/runar-compiler/src/passes/05-stack-lower.ts +++ b/packages/runar-compiler/src/passes/05-stack-lower.ts @@ -23,6 +23,9 @@ import { UnknownANFKindError, MERGED_LOCAL_TEMP_PREFIX } from 'runar-ir-schema'; import { emitVerifySLHDSA } from './slh-dsa-codegen.js'; import { emitVerifyWOTS } from './wots-codegen.js'; import { emitVerifyRabinSig } from './rabin-codegen.js'; +import type { EcCodegenOptions } from './ec-codegen.js'; +import { estimateScriptBytes } from '../metrics/cost-model.js'; +import { optimizeStackIR } from '../optimizer/peephole.js'; import { emitEcAdd, emitEcMul, emitEcMulGen, emitEcNegate, emitEcOnCurve, emitEcModReduce, emitEcEncodeCompressed, @@ -62,6 +65,116 @@ import { const MAX_STACK_DEPTH = 800; +/** + * Experimental lowering options. + * + * Every field defaults to the shipping behaviour, so `lowerToStack(anf)` with + * no options is byte-identical to what the seven tiers produce today. These + * exist so a size experiment can be measured against the default without + * moving a single golden. See `docs/experiments/`. + */ +export interface LoweringOptions { + /** + * Park each curve's field prime / group order in a stack slot inside the EC + * codegen modules instead of re-pushing the 33- or 49-byte literal at every + * modular reduction. + */ + ecConstantPool?: boolean; + + /** + * Operand scheduling strategy. + * + * `'current'` (default) is the shipping behaviour: results are pushed on top + * of the operands that produced them, so a chain that reads the same values + * repeatedly pays a deeper `OP_PICK` each time. + * + * `'liveness'` parks a result on the alt stack when the next binding does + * not consume it, keeping hot operands at depth 0/1, and restores the whole + * spill group before the first binding that needs any of it. + */ + schedulerMode?: 'current' | 'liveness'; +} + +/** + * ANF value kinds the spill scheduler is allowed to reason about. + * + * Deliberately narrow: these compute a single value from operands already on + * the stack and have no control flow, no side effects, and no bespoke stack + * choreography. Everything else — `if`, `loop`, `assert`, state ops, calls, + * `raw_script` — forces the alt stack to be drained first, so none of the + * delicate branch/loop reconciliation in `lowerIf` / `lowerLoop` ever sees a + * non-empty alt stack. + */ +const SPILLABLE_KINDS = new Set([ + 'bin_op', 'unary_op', 'load_const', 'load_param', 'load_prop', +]); + +/** Bytes an OP_TOALTSTACK + OP_FROMALTSTACK round trip costs. */ +const SPILL_ROUND_TRIP_BYTES = 2; + +/** + * Binary operators whose two operands may be materialized in either order. + * + * The operator itself is unchanged — only which operand is brought to the top + * first. Excluded on purpose: + * - `+` is `OP_CAT` when `result_type` is `"bytes"`, and concatenation is not + * commutative, so the caller must check the result type too; + * - `&&` / `||` short-circuit on chain (they lower through a branch, not + * `OP_BOOLAND` / `OP_BOOLOR`), so their operands are not interchangeable + * at this level; + * - `-`, `/`, `%`, shifts and every ordered comparison are order-sensitive. + * + * `===` / `!==` are commutative under both `OP_NUMEQUAL` and `OP_EQUAL`. + */ +const COMMUTATIVE_BINOPS = new Set(['+', '*', '===', '!==', '&', '|', '^']); + +/** + * The ops one `bringToTop` would emit, mirroring its cases exactly, applied to + * a throwaway stack model. + * + * Returns null when the value is not resident, which makes the caller fall + * back to source order rather than guess. + * + * Costing has to go through the real peephole rather than a byte formula, + * because the cheapest-looking local choice is often the one the peephole + * would have erased anyway: two consumed operands at depths 1 and 0 emit + * `OP_SWAP OP_SWAP`, which `swap-swap` deletes outright — free — while + * "cleverly" taking them in the other order emits one real `OP_SWAP` and + * costs a byte. + */ +function materializationOps(model: StackMap, name: string, consume: boolean): StackOp[] | null { + let depth: number; + try { + depth = model.findDepth(name); + } catch { + return null; + } + + const ops: StackOp[] = []; + if (depth === 0) { + if (!consume) { ops.push({ op: 'dup' }); model.dup(); } + return ops; + } + if (depth === 1) { + if (consume) { ops.push({ op: 'swap' }); model.swap(); } + else { ops.push({ op: 'over' }); model.push(model.peekAtDepth(1)); } + return ops; + } + if (consume) { + if (depth === 2) { + ops.push({ op: 'rot' }); + } else { + ops.push({ op: 'push', value: BigInt(depth) }, { op: 'roll', depth }); + } + model.push(model.removeAtDepth(depth)); + } else { + ops.push({ op: 'push', value: BigInt(depth) }, { op: 'pick', depth }); + model.push(model.peekAtDepth(depth)); + } + return ops; +} + + /** * Local hex-to-Uint8Array helper. Avoids a runar-testing dependency * (runar-testing depends on runar-compiler, so the reverse direction @@ -575,11 +688,25 @@ class LoweringContext { */ private readonly renamedParams: Map = new Map(); + /** Experimental lowering options; all default to shipping behaviour. */ + private readonly opts: LoweringOptions; + + /** + * Values parked on the alt stack by the liveness scheduler, bottom -> top. + * + * Always empty in `'current'` mode, and always drained before any binding + * the scheduler does not model (see SPILLABLE_KINDS) and at the end of + * every `lowerBindings` scope. + */ + private altSpills: string[] = []; + constructor( params: string[], properties: ANFProperty[], privateMethods: Map = new Map(), + opts: LoweringOptions = {}, ) { + this.opts = opts; // Parameters are pushed onto the stack by the Bitcoin VM in order. // The first parameter is at the bottom, last parameter at the top. this.stackMap = new StackMap(params); @@ -1035,6 +1162,131 @@ class LoweringContext { * If `consume` is true, use ROLL (removes from original position). * If `consume` is false, use PICK (copies, leaving original in place). */ + // -- liveness scheduler: alt-stack spilling -------------------------------- + // + // ANF pushes every result on top of the operands that made it, so a chain + // reading the same two values repeatedly buries them one slot deeper per + // binding and pays a `push d; OP_PICK` pair instead of a 1-byte `OP_2DUP`. + // Parking the result on the alt stack keeps the operands hot. + // + // Restores are done as a GROUP, not one at a time. Popping the whole alt + // stack puts the values back on the main stack in production order (first + // spilled ends up on top), which is the order an ANF accumulation chain + // reads them in — so the group restore usually lands operands exactly where + // the next bindings want them, and `bringToTop` fixes up any that differ. + + /** True when the liveness scheduler is active for this context. */ + private get schedulingByLiveness(): boolean { + // Never inside a branch arm: `lowerIf` reconciles arms by MAIN-stack depth + // alone (the Layer B/C invariants), so an arm must neither begin nor end + // with a non-empty alt stack. + return this.opts.schedulerMode === 'liveness' && !this._insideBranch; + } + + /** Move the top-of-stack value to the alt stack. */ + private spillToAlt(name: string): void { + this.emitOp({ op: 'opcode', code: 'OP_TOALTSTACK' }); + this.stackMap.pop(); + this.altSpills.push(name); + } + + /** Pop every spilled value back. First-spilled ends up on top. */ + private restoreSpills(): void { + while (this.altSpills.length > 0) { + const name = this.altSpills.pop()!; + this.emitOp({ op: 'opcode', code: 'OP_FROMALTSTACK' }); + this.stackMap.push(name); + this.trackDepth(); + } + } + + /** + * Decide whether the just-computed `name` (now on top) should be parked. + * + * Spilling pays only if the value would otherwise sit above operands that + * get read before it is needed. The estimate counts those intervening reads: + * each one would cross this slot, and crossing it costs at least a byte once + * the access stops being a depth-0/1 `OP_DUP`/`OP_OVER`. Break-even is the + * 2-byte round trip, so 2 intervening reads are required. + */ + private maybeSpill( + binding: ANFBinding, + bindingIndex: number, + bindings: ANFBinding[], + lastUses: Map, + ): void { + if (!this.schedulingByLiveness) return; + if (!SPILLABLE_KINDS.has(binding.value.kind)) return; + // Only a value we actually left on top, under its own name. + if (this.stackMap.depth === 0) return; + if (this.stackMap.peekAtDepth(0) !== binding.name) return; + + const nextUse = this.nextUseAfter(binding.name, bindingIndex, bindings); + if (nextUse === null) return; // dead or used past this scope + if (nextUse === bindingIndex + 1) return; // consumed immediately: no burial + if (lastUses.get(binding.name) !== undefined + && lastUses.get(binding.name)! >= bindings.length) return; // pinned for an outer scope + + // No control flow may follow the spill anywhere in this scope. + // + // Restoring immediately before an `if` leaves the parent stack in a shape + // `lowerIf` was not written for: its arm reconciliation, declared-result + // trim and Layer B/C depth invariants all reason about a main stack the + // scheduler has not been rearranging underneath them. That combination + // MISCOMPILED `if-without-else-multi-temp` — the script kept running and + // started ACCEPTING a witness the shipping compiler rejects, which the + // conformance witness corpus caught (see + // `packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts`). + // Rather than try to make the two agree, spilling stays out of any scope + // that still has control flow ahead of it. `assert` is allowed: it consumes + // a value and emits OP_VERIFY without reshaping anything. + for (let j = bindingIndex + 1; j < bindings.length; j++) { + const kind = bindings[j]!.value.kind; + if (!SPILLABLE_KINDS.has(kind) && kind !== 'assert') return; + } + + // Everything between here and the use must itself be schedulable, or the + // restore would land in the middle of a construct we do not model. + let interveningReads = 0; + for (let j = bindingIndex + 1; j < nextUse; j++) { + const v = bindings[j]!.value; + if (!SPILLABLE_KINDS.has(v.kind)) return; + interveningReads += collectRefs(v).filter(r => r !== binding.name).length; + } + if (interveningReads < SPILL_ROUND_TRIP_BYTES) return; + + // Do not spill a value the very next binding will restore anyway: the + // round trip would be emitted and immediately undone. This is the + // `OP_DIV OP_TOALTSTACK OP_FROMALTSTACK` shape — 2 bytes for nothing. + const next = bindings[bindingIndex + 1]; + if (next !== undefined) { + const nextRefs = collectRefs(next.value); + if (!SPILLABLE_KINDS.has(next.value.kind) + || nextRefs.some(r => this.altSpills.includes(r))) return; + } + + this.spillToAlt(binding.name); + } + + /** Index of the first binding at or after `from + 1` that references `name`. */ + private nextUseAfter(name: string, from: number, bindings: ANFBinding[]): number | null { + for (let j = from + 1; j < bindings.length; j++) { + if (collectRefs(bindings[j]!.value).includes(name)) return j; + } + return null; + } + + /** + * Options handed to the EC / NIST codegen modules. + * + * Returns `undefined` — not `{}` — when nothing is enabled, so the emitters + * take their untouched default path and the emitted bytes are provably + * identical to the shipping ones. + */ + private ecCodegenOptions(): EcCodegenOptions | undefined { + return this.opts.ecConstantPool ? { constantPool: true } : undefined; + } + bringToTop(name: string, consume: boolean): void { const depth = this.stackMap.findDepth(name); @@ -1178,6 +1430,16 @@ class LoweringContext { for (let i = 0; i < bindings.length; i++) { const binding = bindings[i]!; + // Drain the alt stack before anything the scheduler does not model, and + // before the first binding that reads a parked value. + if (this.altSpills.length > 0) { + const refs = collectRefs(binding.value); + if (!SPILLABLE_KINDS.has(binding.value.kind) + || refs.some(r => this.altSpills.includes(r)) + || i === lastAssertIdx || i === terminalIfIdx) { + this.restoreSpills(); + } + } // Propagate source location from ANF binding to StackOps this.currentSourceLoc = binding.sourceLoc; if (binding.value.kind === 'assert' && i === lastAssertIdx) { @@ -1188,10 +1450,13 @@ class LoweringContext { this.lowerIf(binding.name, binding.value.cond, binding.value.then, binding.value.else, binding.value.results ?? [], i, lastUses, true); } else { this.lowerBinding(binding, i, lastUses); + this.maybeSpill(binding, i, bindings, lastUses); } this.currentSourceLoc = undefined; } + // Nothing may outlive the scope on the alt stack. + this.restoreSpills(); } private lowerBinding( @@ -1494,6 +1759,45 @@ class LoweringContext { } } + /** + * Would materializing a commutative operator's operands right-then-left cost + * fewer bytes than left-then-right? + * + * Scored with the exact emit-time cost of each `bringToTop`, on a throwaway + * copy of the stack map so the real one is untouched. Ties keep source + * order, which is what makes `'current'` and `'liveness'` identical wherever + * this cannot help. + */ + private shouldSwapOperands( + op: string, + left: string, + right: string, + leftConsume: boolean, + rightConsume: boolean, + resultType?: string, + ): boolean { + if (this.opts.schedulerMode !== 'liveness') return false; + if (!COMMUTATIVE_BINOPS.has(op)) return false; + // `+` on ByteString operands is OP_CAT, which is not commutative. + if (resultType === 'bytes' && op === '+') return false; + if (left === right) return false; + + const cost = (first: string, firstConsume: boolean, second: string, secondConsume: boolean): number => { + const model = this.stackMap.clone(); + const ops: StackOp[] = []; + for (const [name, consume] of [[first, firstConsume], [second, secondConsume]] as const) { + const emitted = materializationOps(model, name, consume); + if (emitted === null) return Number.POSITIVE_INFINITY; // not resident + ops.push(...emitted); + } + // Score what the emitter would actually see, peephole included. + return estimateScriptBytes(optimizeStackIR(ops)); + }; + + return cost(right, rightConsume, left, leftConsume) + < cost(left, leftConsume, right, rightConsume); + } + private lowerBinOp( bindingName: string, op: string, @@ -1503,13 +1807,20 @@ class LoweringContext { lastUses: Map, resultType?: string, ): void { - // Get left operand to stack first const leftConsume = this.operandConsume(left, [left, right], bindingIndex, lastUses); - this.bringToTop(left, leftConsume); - - // Get right operand to stack const rightConsume = this.operandConsume(right, [left, right], bindingIndex, lastUses); - this.bringToTop(right, rightConsume); + + // Commutative operators may take their operands in either order, so pick + // the cheaper arrangement. The common win: the operand already on top is + // materialized SECOND for free, instead of being buried by the other one + // and then swapped back. + if (this.shouldSwapOperands(op, left, right, leftConsume, rightConsume, resultType)) { + this.bringToTop(right, rightConsume); + this.bringToTop(left, leftConsume); + } else { + this.bringToTop(left, leftConsume); + this.bringToTop(right, rightConsume); + } // Pop both operands (the opcode consumes them) this.stackMap.pop(); @@ -4901,13 +5212,14 @@ class LoweringContext { for (let i = 0; i < args.length; i++) this.stackMap.pop(); const emitFn = (op: StackOp) => this.emitOp(op); + const ecOpts = this.ecCodegenOptions(); switch (func) { - case 'ecAdd': emitEcAdd(emitFn); break; - case 'ecMul': emitEcMul(emitFn); break; - case 'ecMulGen': emitEcMulGen(emitFn); break; - case 'ecNegate': emitEcNegate(emitFn); break; - case 'ecOnCurve': emitEcOnCurve(emitFn); break; + case 'ecAdd': emitEcAdd(emitFn, ecOpts); break; + case 'ecMul': emitEcMul(emitFn, ecOpts); break; + case 'ecMulGen': emitEcMulGen(emitFn, ecOpts); break; + case 'ecNegate': emitEcNegate(emitFn, ecOpts); break; + case 'ecOnCurve': emitEcOnCurve(emitFn, ecOpts); break; case 'ecModReduce': emitEcModReduce(emitFn); break; case 'ecEncodeCompressed': emitEcEncodeCompressed(emitFn); break; case 'ecMakePoint': emitEcMakePoint(emitFn); break; @@ -4938,19 +5250,20 @@ class LoweringContext { for (let i = 0; i < args.length; i++) this.stackMap.pop(); const emitFn = (op: StackOp) => this.emitOp(op); + const ecOpts = this.ecCodegenOptions(); switch (func) { - case 'p256Add': emitP256Add(emitFn); break; - case 'p256Mul': emitP256Mul(emitFn); break; - case 'p256MulGen': emitP256MulGen(emitFn); break; - case 'p256Negate': emitP256Negate(emitFn); break; - case 'p256OnCurve': emitP256OnCurve(emitFn); break; + case 'p256Add': emitP256Add(emitFn, ecOpts); break; + case 'p256Mul': emitP256Mul(emitFn, ecOpts); break; + case 'p256MulGen': emitP256MulGen(emitFn, ecOpts); break; + case 'p256Negate': emitP256Negate(emitFn, ecOpts); break; + case 'p256OnCurve': emitP256OnCurve(emitFn, ecOpts); break; case 'p256EncodeCompressed': emitP256EncodeCompressed(emitFn); break; - case 'p384Add': emitP384Add(emitFn); break; - case 'p384Mul': emitP384Mul(emitFn); break; - case 'p384MulGen': emitP384MulGen(emitFn); break; - case 'p384Negate': emitP384Negate(emitFn); break; - case 'p384OnCurve': emitP384OnCurve(emitFn); break; + case 'p384Add': emitP384Add(emitFn, ecOpts); break; + case 'p384Mul': emitP384Mul(emitFn, ecOpts); break; + case 'p384MulGen': emitP384MulGen(emitFn, ecOpts); break; + case 'p384Negate': emitP384Negate(emitFn, ecOpts); break; + case 'p384OnCurve': emitP384OnCurve(emitFn, ecOpts); break; case 'p384EncodeCompressed': emitP384EncodeCompressed(emitFn); break; default: throw new Error(`Unknown NIST EC builtin: ${func}`); } @@ -4975,8 +5288,9 @@ class LoweringContext { this.stackMap.pop(); // sig this.stackMap.pop(); // msg const emitFn = (op: StackOp) => this.emitOp(op); - if (func === 'verifyECDSA_P256') emitVerifyECDSA_P256(emitFn); - else emitVerifyECDSA_P384(emitFn); + const ecOpts = this.ecCodegenOptions(); + if (func === 'verifyECDSA_P256') emitVerifyECDSA_P256(emitFn, ecOpts); + else emitVerifyECDSA_P384(emitFn, ecOpts); this.stackMap.push(bindingName); this.trackDepth(); } @@ -5452,7 +5766,7 @@ function hexToBytes(hex: string): Uint8Array { * named temporaries via a stack map and emits PICK/ROLL to materialise * values as needed. */ -export function lowerToStack(program: ANFProgram): StackProgram { +export function lowerToStack(program: ANFProgram, opts: LoweringOptions = {}): StackProgram { const methods: StackMethod[] = []; const privateMethods = new Map(); @@ -5466,7 +5780,7 @@ export function lowerToStack(program: ANFProgram): StackProgram { if (method.name !== 'constructor' && !method.isPublic) { continue; } - const stackMethod = lowerMethod(method, program.properties, privateMethods); + const stackMethod = lowerMethod(method, program.properties, privateMethods, opts); methods.push(stackMethod); } @@ -5562,10 +5876,43 @@ function methodUsesCodePart(bindings: ANFBinding[]): boolean { return false; } +/** + * Lower one method, and — under `schedulerMode: 'liveness'` — pick the + * cheaper of the two schedules by emitted bytes. + * + * The per-site spill heuristic in `maybeSpill` is greedy and approximate: it + * cannot know whether removing one slot actually moves an access across a + * cost boundary, because `OP_DUP`/`OP_SWAP`/`OP_OVER`/`OP_ROT` all cost one + * byte, so burying a value is free until the access reaches depth 3. Rather + * than model the whole stack evolution ahead of time, lower both ways and let + * the exact cost model decide. That makes "the scheduler never grows a method" + * a structural property rather than a hope. + * + * Sizes are compared AFTER the peephole pass, since that is what the emitter + * finally sees — spilling changes which peephole rules fire (an `OP_OVER + * OP_OVER` that fused into `OP_2DUP` may become a plain `OP_DUP`). + */ function lowerMethod( method: ANFMethod, properties: ANFProperty[], privateMethods: Map, + opts: LoweringOptions = {}, +): StackMethod { + if (opts.schedulerMode === 'liveness') { + const scheduled = lowerMethodOnce(method, properties, privateMethods, opts); + const baseline = lowerMethodOnce(method, properties, privateMethods, + { ...opts, schedulerMode: 'current' }); + const size = (m: StackMethod): number => estimateScriptBytes(optimizeStackIR(m.ops)); + return size(scheduled) < size(baseline) ? scheduled : baseline; + } + return lowerMethodOnce(method, properties, privateMethods, opts); +} + +function lowerMethodOnce( + method: ANFMethod, + properties: ANFProperty[], + privateMethods: Map, + opts: LoweringOptions = {}, ): StackMethod { const paramNames = method.params.map(p => p.name); @@ -5588,7 +5935,7 @@ function lowerMethod( paramNames.unshift('_codePart'); } - const ctx = new LoweringContext(paramNames, properties, privateMethods); + const ctx = new LoweringContext(paramNames, properties, privateMethods, opts); // Pass terminalAssert=true for public methods so the last assert leaves // its value on the stack (Bitcoin Script requires a truthy top-of-stack). ctx.lowerBindings(method.body, method.isPublic); diff --git a/packages/runar-compiler/src/passes/ec-codegen.ts b/packages/runar-compiler/src/passes/ec-codegen.ts index e1dc017f..566c5d00 100644 --- a/packages/runar-compiler/src/passes/ec-codegen.ts +++ b/packages/runar-compiler/src/passes/ec-codegen.ts @@ -10,6 +10,7 @@ */ import type { StackOp } from '../ir/index.js'; +import { sizeOfPushValue } from '../metrics/cost-model.js'; // =========================================================================== // Constants @@ -40,13 +41,49 @@ function bigintToBytes32(n: bigint): Uint8Array { // ECTracker — named stack state tracker (mirrors SLHTracker) // =========================================================================== +/** + * Codegen options shared by every EC / NIST-curve emitter. + * + * Off by default: with no options (or `constantPool: false`) each emitter is + * byte-identical to what the seven tiers ship today, so no golden, size + * baseline, or cross-tier parity gate can move. + */ +export interface EcCodegenOptions { + /** + * Park large repeated constants (the field prime, the group order) in a + * stack slot and copy them with `OP_PICK` instead of re-pushing the literal. + * + * `fieldMod` pushes the 256-bit prime at every modular reduction — 34 bytes + * a time, 20,025 times in `p256-wallet` (71 % of that fixture). A pick from + * a slot a dozen deep costs 2. See + * `docs/experiments/script-size-optimization-baseline.md`. + */ + constantPool?: boolean; +} + +/** Stack slot names reserved for pooled constants. */ +export const POOL_FIELD_P = '_pool$p'; +export const POOL_GROUP_N = '_pool$n'; + export class ECTracker { nm: (string | null)[]; _e: (op: StackOp) => void; - - constructor(init: (string | null)[], emit: (op: StackOp) => void) { + /** True when this tracker may serve constants from a pooled slot. */ + readonly pooling: boolean; + + constructor( + init: (string | null)[], + emit: (op: StackOp) => void, + opts?: EcCodegenOptions, + ) { this.nm = [...init]; this._e = emit; + this.pooling = opts?.constantPool === true; + } + + /** The options this tracker was built with, for handing to a nested tracker. */ + get options(): EcCodegenOptions { + return { constantPool: this.pooling }; } get depth(): number { return this.nm.length; } @@ -109,6 +146,48 @@ export class ECTracker { } toTop(name: string): void { this.roll(this.findDepth(name)); } copyToTop(name: string, n?: string): void { this.pick(this.findDepth(name), n ?? name); } + + // -- constant pool -------------------------------------------------------- + // + // A pooled constant is an ordinary tracked slot; nothing about the stack + // model changes. `pushConst` just chooses, per call site and by emitted + // bytes, between copying that slot and re-pushing the literal. Nested + // trackers built with `[...t.nm]` inherit the slot for free, so pooled + // constants work unchanged inside an `OP_IF` arm. + + /** Park `value` in `slot` for the lifetime of this emitter. No-op when pooling is off. */ + poolConstant(slot: string, value: bigint): void { + if (!this.pooling || this.nm.includes(slot)) return; + this.pushInt(slot, value); + } + + /** Remove a pooled slot. No-op when pooling is off or the slot is absent. */ + releaseConstant(slot: string): void { + if (!this.pooling || !this.nm.includes(slot)) return; + this.toTop(slot); + this.drop(); + } + + /** + * Materialize `value` on top as `name`, from the pooled `slot` when that is + * cheaper in emitted bytes than pushing the literal. + * + * The comparison is exact — `sizeOfPushValue` is the same encoder the emit + * pass uses — so pooling can never make a call site bigger. A pick at depth + * d costs `sizeOfPushValue(d) + 1`; depths 0 and 1 are OP_DUP / OP_OVER, 1 + * byte each. + */ + pushConst(slot: string, value: bigint, name: string): void { + if (this.pooling && this.nm.includes(slot)) { + const d = this.findDepth(slot); + const pickCost = d <= 1 ? 1 : sizeOfPushValue(BigInt(d)) + 1; + if (pickCost < sizeOfPushValue(value)) { + this.pick(d, name); + return; + } + } + this.pushInt(name, value); + } toAlt(): void { this.op('OP_TOALTSTACK'); this.nm.pop(); } fromAlt(n: string): void { this.op('OP_FROMALTSTACK'); this.nm.push(n); } rename(n: string): void { @@ -145,11 +224,10 @@ export class ECTracker { /** Push the field prime p onto the stack as a script number. */ function pushFieldP(t: ECTracker, name: string): void { - // Push p directly as a BigInt — the emit pass encodes it as a proper - // little-endian sign-magnitude script number push. - t.pushInt(name, FIELD_P); + t.pushConst(POOL_FIELD_P, FIELD_P, name); } + /** * Reduce a scalar to [0, n-1]: ((k mod n) + n) mod n. * @@ -165,7 +243,7 @@ function pushFieldP(t: ECTracker, name: string): void { * well defined. */ function emitScalarReduce(t: ECTracker, kName: string, resultName: string): void { - t.pushInt('_n_red', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n_red'); t.rawBlock([kName, '_n_red'], resultName, (e) => { e({ op: 'opcode', code: 'OP_2DUP' }); e({ op: 'opcode', code: 'OP_MOD' }); @@ -626,7 +704,7 @@ function jacobianToAffine(t: ECTracker, rxName: string, ryName: string): void { */ function buildJacobianAddAffineInline(e: (op: StackOp) => void, t: ECTracker): void { // Create inner tracker with cloned stack state - jacobianAddAffineBody(new ECTracker([...t.nm], e), false); + jacobianAddAffineBody(new ECTracker([...t.nm], e, t.options), false); } /** @@ -780,7 +858,7 @@ function selectCoord(t: ECTracker, addName: string, dblName: string, condName: s * Stack layout: [..., ax, ay, _k, jx, jy, jz] — same in and out. */ function buildJacobianAddOrDoubleInline(e: (op: StackOp) => void, t: ECTracker): void { - const it = new ECTracker([...t.nm], e); + const it = new ECTracker([...t.nm], e, t.options); // Keep the pre-add accumulator: it is what must be DOUBLED in the // exceptional case, and the add below consumes jx/jy/jz. @@ -839,12 +917,14 @@ function buildJacobianAddOrDoubleInline(e: (op: StackOp) => void, t: ECTracker): * Stack in: [point_a, point_b] (b on top) * Stack out: [result_point] */ -export function emitEcAdd(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pa', '_pb'], emit); +export function emitEcAdd(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pa', '_pb'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); decomposePoint(t, '_pa', 'px', 'py'); decomposePoint(t, '_pb', 'qx', 'qy'); affineAdd(t); composePoint(t, 'rx', 'ry', '_result'); + t.releaseConstant(POOL_FIELD_P); } /** @@ -858,8 +938,10 @@ export function emitEcAdd(emit: (op: StackOp) => void): void { * This avoids the k+n overflow issue where bit 256 was only set for * large k, causing incorrect results for ~half of all scalar values. */ -export function emitEcMul(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt', '_k'], emit); +export function emitEcMul(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt', '_k'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); + t.poolConstant(POOL_GROUP_N, CURVE_N); decomposePoint(t, '_pt', 'ax', 'ay'); // k' = k + 3n: guarantees bit 257 is set for MSB-first double-and-add. @@ -870,15 +952,15 @@ export function emitEcMul(emit: (op: StackOp) => void): void { // usually an unlock argument — so reduce it first. See emitScalarReduce. t.toTop('_k'); emitScalarReduce(t, '_k', '_kr'); - t.pushInt('_n', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n'); t.rawBlock(['_kr', '_n'], '_kn', (e) => { e({ op: 'opcode', code: 'OP_ADD' }); }); - t.pushInt('_n2', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n2'); t.rawBlock(['_kn', '_n2'], '_kn2', (e) => { e({ op: 'opcode', code: 'OP_ADD' }); }); - t.pushInt('_n3', CURVE_N); + t.pushConst(POOL_GROUP_N, CURVE_N, '_n3'); t.rawBlock(['_kn2', '_n3'], '_kn3', (e) => { e({ op: 'opcode', code: 'OP_ADD' }); }); @@ -935,6 +1017,8 @@ export function emitEcMul(emit: (op: StackOp) => void): void { t.toTop('_k'); t.drop(); composePoint(t, '_rx', '_ry', '_result'); + t.releaseConstant(POOL_GROUP_N); + t.releaseConstant(POOL_FIELD_P); } /** @@ -942,14 +1026,14 @@ export function emitEcMul(emit: (op: StackOp) => void): void { * Stack in: [scalar] * Stack out: [result_point] */ -export function emitEcMulGen(emit: (op: StackOp) => void): void { +export function emitEcMulGen(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { // Push generator point as 64-byte blob, then delegate to ecMul const gPoint = new Uint8Array(64); gPoint.set(bigintToBytes32(GEN_X), 0); gPoint.set(bigintToBytes32(GEN_Y), 32); emit({ op: 'push', value: gPoint }); emit({ op: 'swap' }); // [point, scalar] - emitEcMul(emit); + emitEcMul(emit, opts); } /** @@ -957,12 +1041,14 @@ export function emitEcMulGen(emit: (op: StackOp) => void): void { * Stack in: [point] * Stack out: [negated_point] */ -export function emitEcNegate(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitEcNegate(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); decomposePoint(t, '_pt', '_nx', '_ny'); pushFieldP(t, '_fp'); fieldSub(t, '_fp', '_ny', '_neg_y'); composePoint(t, '_nx', '_neg_y', '_result'); + t.releaseConstant(POOL_FIELD_P); } /** @@ -970,8 +1056,9 @@ export function emitEcNegate(emit: (op: StackOp) => void): void { * Stack in: [point] * Stack out: [boolean] */ -export function emitEcOnCurve(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitEcOnCurve(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, FIELD_P); decomposePoint(t, '_pt', '_x', '_y'); // GAP-301: coordinate canonicity. `decomposePoint` BIN2NUMs each coordinate @@ -1019,6 +1106,7 @@ export function emitEcOnCurve(emit: (op: StackOp) => void): void { t.rawBlock(['_canon', '_curve_eq'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_FIELD_P); } /** diff --git a/packages/runar-compiler/src/passes/p256-p384-codegen.ts b/packages/runar-compiler/src/passes/p256-p384-codegen.ts index ac45060e..f4a6387a 100644 --- a/packages/runar-compiler/src/passes/p256-p384-codegen.ts +++ b/packages/runar-compiler/src/passes/p256-p384-codegen.ts @@ -14,7 +14,8 @@ */ import type { StackOp } from '../ir/index.js'; -import { ECTracker } from './ec-codegen.js'; +import { ECTracker, POOL_FIELD_P, POOL_GROUP_N } from './ec-codegen.js'; +import type { EcCodegenOptions } from './ec-codegen.js'; // =========================================================================== // P-256 constants (secp256r1 / NIST P-256) @@ -129,7 +130,7 @@ const P384_PARAMS: CurveParams = { }; function pushFieldP(t: ECTracker, name: string, c: CurveParams): void { - t.pushInt(name, c.fieldP); + t.pushConst(POOL_FIELD_P, c.fieldP, name); } function cFieldMod(t: ECTracker, aName: string, resultName: string, c: CurveParams): void { @@ -230,7 +231,7 @@ const P256_GROUP: GroupParams = { n: P256_N, nMinus2: P256_N_MINUS_2 }; const P384_GROUP: GroupParams = { n: P384_N, nMinus2: P384_N_MINUS_2 }; function pushGroupN(t: ECTracker, name: string, g: GroupParams): void { - t.pushInt(name, g.n); + t.pushConst(POOL_GROUP_N, g.n, name); } function cGroupMod(t: ECTracker, aName: string, resultName: string, g: GroupParams): void { @@ -648,7 +649,7 @@ function cJacobianToAffine(t: ECTracker, rxName: string, ryName: string, c: Curv * After: [..., ax, ay, _k, jx', jy', jz'] */ function buildJacobianAddAffineInline(e: (op: StackOp) => void, t: ECTracker, c: CurveParams): void { - jacobianAddAffineBody(new ECTracker([...t.nm], e), false, c); + jacobianAddAffineBody(new ECTracker([...t.nm], e, t.options), false, c); } /** @@ -795,7 +796,7 @@ function cSelectCoord( * Stack layout: [..., ax, ay, _k, jx, jy, jz] — same in and out. */ function buildJacobianAddOrDoubleInline(e: (op: StackOp) => void, t: ECTracker, c: CurveParams): void { - const it = new ECTracker([...t.nm], e); + const it = new ECTracker([...t.nm], e, t.options); // Keep the pre-add accumulator: it is what must be DOUBLED in the // exceptional case, and the add below consumes jx/jy/jz. @@ -862,8 +863,11 @@ function cEmitMul( emit: (op: StackOp) => void, c: CurveParams, g: GroupParams, + opts?: EcCodegenOptions, ): void { - const t = new ECTracker(['_pt', '_k'], emit); + const t = new ECTracker(['_pt', '_k'], emit, opts); + t.poolConstant(POOL_FIELD_P, c.fieldP); + t.poolConstant(POOL_GROUP_N, g.n); cDecomposePoint(t, '_pt', 'ax', 'ay', c); // k' = k + 3n: guarantees a fixed high bit for MSB-first double-and-add. @@ -943,6 +947,8 @@ function cEmitMul( t.toTop('_k'); t.drop(); cComposePoint(t, '_rx', '_ry', '_result', c); + t.releaseConstant(POOL_GROUP_N); + t.releaseConstant(POOL_FIELD_P); } // =========================================================================== @@ -1298,8 +1304,15 @@ function cEmitVerifyECDSA( sqrtExp: bigint, gx: bigint, gy: bigint, + opts?: EcCodegenOptions, ): void { - const t = new ECTracker(['_msg', '_sig', '_pk'], emit); + const t = new ECTracker(['_msg', '_sig', '_pk'], emit, opts); + // The verifier does hundreds of reductions OUTSIDE the two ladders — + // decompression's sqrt ladder, cGroupInv, cAffineAdd, the final cGroupMod. + // Each ladder pools separately: cEmitMul runs on its own tracker that + // deliberately cannot see this stack, so it cannot reach this slot. + t.poolConstant(POOL_FIELD_P, c.fieldP); + t.poolConstant(POOL_GROUP_N, g.n); // Step 0: length gate. `_sig` and `_pk` are bare ByteString in the builtin // table and the type checker imposes no width, so both arrive attacker-sized. @@ -1410,7 +1423,7 @@ function cEmitVerifyECDSA( t.nm.pop(); // _G // Emit the mul (it manages its own tracker internally) - cEmitMul(emit, c, g); + cEmitMul(emit, c, g, opts); // After mul, one result point is on the stack t.nm.push('_R1_point'); @@ -1433,7 +1446,7 @@ function cEmitVerifyECDSA( // Pop from tracker, emit mul, push result t.nm.pop(); // _u2 t.nm.pop(); // _Q_point - cEmitMul(emit, c, g); + cEmitMul(emit, c, g, opts); t.nm.push('_R2_point'); // Restore R1 point @@ -1487,6 +1500,8 @@ function cEmitVerifyECDSA( t.rawBlock(['_input_ok', '_sig_ok'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_GROUP_N); + t.releaseConstant(POOL_FIELD_P); } // =========================================================================== @@ -1498,12 +1513,14 @@ function cEmitVerifyECDSA( * Stack in: [P256Point, P256Point] (second on top) * Stack out: [P256Point] */ -export function emitP256Add(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pa', '_pb'], emit); +export function emitP256Add(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pa', '_pb'], emit, opts); + t.poolConstant(POOL_FIELD_P, P256_PARAMS.fieldP); cDecomposePoint(t, '_pa', 'px', 'py', P256_PARAMS); cDecomposePoint(t, '_pb', 'qx', 'qy', P256_PARAMS); cAffineAdd(t, P256_PARAMS); cComposePoint(t, 'rx', 'ry', '_result', P256_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1511,8 +1528,8 @@ export function emitP256Add(emit: (op: StackOp) => void): void { * Stack in: [P256Point, bigint] (scalar on top) * Stack out: [P256Point] */ -export function emitP256Mul(emit: (op: StackOp) => void): void { - cEmitMul(emit, P256_PARAMS, P256_GROUP); +export function emitP256Mul(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitMul(emit, P256_PARAMS, P256_GROUP, opts); } /** @@ -1520,13 +1537,13 @@ export function emitP256Mul(emit: (op: StackOp) => void): void { * Stack in: [bigint] * Stack out: [P256Point] */ -export function emitP256MulGen(emit: (op: StackOp) => void): void { +export function emitP256MulGen(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { const gPoint = new Uint8Array(64); gPoint.set(bigintToBytes(P256_GX, 32), 0); gPoint.set(bigintToBytes(P256_GY, 32), 32); emit({ op: 'push', value: gPoint }); emit({ op: 'swap' }); // [point, scalar] - emitP256Mul(emit); + emitP256Mul(emit, opts); } /** @@ -1534,12 +1551,14 @@ export function emitP256MulGen(emit: (op: StackOp) => void): void { * Stack in: [P256Point] * Stack out: [P256Point] */ -export function emitP256Negate(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP256Negate(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P256_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_nx', '_ny', P256_PARAMS); pushFieldP(t, '_fp', P256_PARAMS); cFieldSub(t, '_fp', '_ny', '_neg_y', P256_PARAMS); cComposePoint(t, '_nx', '_neg_y', '_result', P256_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1547,8 +1566,9 @@ export function emitP256Negate(emit: (op: StackOp) => void): void { * Stack in: [P256Point] * Stack out: [boolean] */ -export function emitP256OnCurve(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP256OnCurve(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P256_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_x', '_y', P256_PARAMS); cEmitCanonicityGuard(t, '_x', '_y', P256_PARAMS); @@ -1578,6 +1598,7 @@ export function emitP256OnCurve(emit: (op: StackOp) => void): void { t.rawBlock(['_canon', '_curve_eq'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1616,8 +1637,8 @@ export function emitP256EncodeCompressed(emit: (op: StackOp) => void): void { * Stack in: [msg_bytes, sig(64B), pubkey(33B)] (pubkey on top) * Stack out: [boolean] */ -export function emitVerifyECDSA_P256(emit: (op: StackOp) => void): void { - cEmitVerifyECDSA(emit, P256_PARAMS, P256_GROUP, P256_B, P256_SQRT_EXP, P256_GX, P256_GY); +export function emitVerifyECDSA_P256(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitVerifyECDSA(emit, P256_PARAMS, P256_GROUP, P256_B, P256_SQRT_EXP, P256_GX, P256_GY, opts); } // =========================================================================== @@ -1629,12 +1650,14 @@ export function emitVerifyECDSA_P256(emit: (op: StackOp) => void): void { * Stack in: [P384Point, P384Point] (second on top) * Stack out: [P384Point] */ -export function emitP384Add(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pa', '_pb'], emit); +export function emitP384Add(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pa', '_pb'], emit, opts); + t.poolConstant(POOL_FIELD_P, P384_PARAMS.fieldP); cDecomposePoint(t, '_pa', 'px', 'py', P384_PARAMS); cDecomposePoint(t, '_pb', 'qx', 'qy', P384_PARAMS); cAffineAdd(t, P384_PARAMS); cComposePoint(t, 'rx', 'ry', '_result', P384_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1642,8 +1665,8 @@ export function emitP384Add(emit: (op: StackOp) => void): void { * Stack in: [P384Point, bigint] (scalar on top) * Stack out: [P384Point] */ -export function emitP384Mul(emit: (op: StackOp) => void): void { - cEmitMul(emit, P384_PARAMS, P384_GROUP); +export function emitP384Mul(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitMul(emit, P384_PARAMS, P384_GROUP, opts); } /** @@ -1651,13 +1674,13 @@ export function emitP384Mul(emit: (op: StackOp) => void): void { * Stack in: [bigint] * Stack out: [P384Point] */ -export function emitP384MulGen(emit: (op: StackOp) => void): void { +export function emitP384MulGen(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { const gPoint = new Uint8Array(96); gPoint.set(bigintToBytes(P384_GX, 48), 0); gPoint.set(bigintToBytes(P384_GY, 48), 48); emit({ op: 'push', value: gPoint }); emit({ op: 'swap' }); // [point, scalar] - emitP384Mul(emit); + emitP384Mul(emit, opts); } /** @@ -1665,12 +1688,14 @@ export function emitP384MulGen(emit: (op: StackOp) => void): void { * Stack in: [P384Point] * Stack out: [P384Point] */ -export function emitP384Negate(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP384Negate(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P384_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_nx', '_ny', P384_PARAMS); pushFieldP(t, '_fp', P384_PARAMS); cFieldSub(t, '_fp', '_ny', '_neg_y', P384_PARAMS); cComposePoint(t, '_nx', '_neg_y', '_result', P384_PARAMS); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1678,8 +1703,9 @@ export function emitP384Negate(emit: (op: StackOp) => void): void { * Stack in: [P384Point] * Stack out: [boolean] */ -export function emitP384OnCurve(emit: (op: StackOp) => void): void { - const t = new ECTracker(['_pt'], emit); +export function emitP384OnCurve(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + const t = new ECTracker(['_pt'], emit, opts); + t.poolConstant(POOL_FIELD_P, P384_PARAMS.fieldP); cDecomposePoint(t, '_pt', '_x', '_y', P384_PARAMS); cEmitCanonicityGuard(t, '_x', '_y', P384_PARAMS); @@ -1709,6 +1735,7 @@ export function emitP384OnCurve(emit: (op: StackOp) => void): void { t.rawBlock(['_canon', '_curve_eq'], '_result', (e) => { e({ op: 'opcode', code: 'OP_BOOLAND' }); }); + t.releaseConstant(POOL_FIELD_P); } /** @@ -1747,6 +1774,6 @@ export function emitP384EncodeCompressed(emit: (op: StackOp) => void): void { * Stack in: [msg_bytes, sig(96B), pubkey(49B)] (pubkey on top) * Stack out: [boolean] */ -export function emitVerifyECDSA_P384(emit: (op: StackOp) => void): void { - cEmitVerifyECDSA(emit, P384_PARAMS, P384_GROUP, P384_B, P384_SQRT_EXP, P384_GX, P384_GY); +export function emitVerifyECDSA_P384(emit: (op: StackOp) => void, opts?: EcCodegenOptions): void { + cEmitVerifyECDSA(emit, P384_PARAMS, P384_GROUP, P384_B, P384_SQRT_EXP, P384_GX, P384_GY, opts); } diff --git a/packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts b/packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts new file mode 100644 index 00000000..7ab6abad --- /dev/null +++ b/packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts @@ -0,0 +1,251 @@ +/** + * EC constant pooling — semantic equivalence on the real interpreter. + * + * Pooling replaces ~20,000 inline pushes of a curve's field prime with picks + * from one resident stack slot (see + * `docs/experiments/script-size-optimization-baseline.md`). It changes stack + * layout inside every EC emitter, including inside `OP_IF` arms, so "the byte + * count went down" is not evidence of anything on its own. + * + * Two kinds of proof here, both through @bsv/sdk's `Spend`: + * + * 1. DIFFERENTIAL — for the same inputs, the pooled and unpooled scripts leave + * an identical stack. No oracle needed and no fixture to get wrong: the + * unpooled emitter is the specification. + * 2. ORACLE — `verifyECDSA_*` accepts a genuine OpenSSL signature and rejects + * every near-miss, under BOTH variants. This is the one that would catch a + * pooled slot being read where a *different* value was intended, which a + * pure differential over random inputs can miss if both variants are wrong + * in the same way (they cannot be here — only one of them was changed — + * but the reject cases also pin the security-relevant behaviour). + * + * Max stack depth is measured, not assumed: pooling adds resident slots, and + * the interpreter's 1,000-element budget is the real limit. + */ + +import { describe, it, expect } from 'vitest'; +import { createSign, generateKeyPairSync } from 'node:crypto'; +import { + emitMethod, + emitVerifyECDSA_P256, emitVerifyECDSA_P384, + emitP256Add, emitP256Mul, emitP256Negate, emitP256OnCurve, + emitP384Add, emitP384Negate, emitP384OnCurve, + emitEcAdd, emitEcMul, emitEcNegate, emitEcOnCurve, +} from 'runar-compiler'; +import type { StackOp } from 'runar-ir-schema'; +import { ScriptVM } from '../index.js'; + +type Emitter = (emit: (op: StackOp) => void, opts?: { constantPool?: boolean }) => void; + +const blob = (hex: string) => Uint8Array.from(Buffer.from(hex, 'hex')); + +interface RunResult { + stack: string[]; + error: string | null; + maxStackDepth: number; +} + +/** Emit `inputs` then the emitter's body, and execute the whole thing. */ +function run(emitter: Emitter, inputs: StackOp[], pooled: boolean): RunResult { + const ops: StackOp[] = [...inputs]; + emitter(op => ops.push(op), pooled ? { constantPool: true } : undefined); + const { scriptHex } = emitMethod({ name: 't', ops } as never) as { scriptHex: string }; + const r = new ScriptVM().executeHex(scriptHex) as never as { + stack: Uint8Array[]; error?: string; maxStackDepth: number; + }; + return { + stack: r.stack.map(b => Buffer.from(b).toString('hex')), + error: r.error ?? null, + maxStackDepth: r.maxStackDepth, + }; +} + +/** Assert both variants agree completely, and report the depth cost. */ +function expectSame(emitter: Emitter, inputs: StackOp[]): { off: RunResult; on: RunResult } { + const off = run(emitter, inputs, false); + const on = run(emitter, inputs, true); + expect(on.error).toBe(off.error); + expect(on.stack).toEqual(off.stack); + return { off, on }; +} + +const push = (hex: string): StackOp => ({ op: 'push', value: blob(hex) } as StackOp); +const pushN = (n: bigint): StackOp => ({ op: 'push', value: n } as StackOp); + +// --------------------------------------------------------------------------- +// Curve fixtures +// --------------------------------------------------------------------------- + +const P256 = { + p: 0xffffffff00000001000000000000000000000000ffffffffffffffffffffffffn, + n: 0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551n, + gx: 0x6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296n, + gy: 0x4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5n, + bytes: 32, +}; +const SECP = { + p: 0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc2fn, + n: 0xfffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141n, + gx: 0x79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798n, + gy: 0x483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8n, + bytes: 32, +}; +const P384 = { + gx: 0xaa87ca22be8b05378eb1c71ef320ad746e1d3b628ba79b9859f741e082542a385502f25dbf55296c3a545e3872760ab7n, + gy: 0x3617de4a96262c6f5d9e98bf9292dc29f8f41dbd289a147ce9da3113b5f0b8c00a60b1ce1d7e819d7a431d7c90ea0e5fn, + bytes: 48, +}; + +const hx = (v: bigint, bytes: number) => v.toString(16).padStart(bytes * 2, '0'); +const point = (x: bigint, y: bigint, bytes: number) => hx(x, bytes) + hx(y, bytes); + +// --------------------------------------------------------------------------- +// 1. Differential — the unpooled emitter is the specification +// --------------------------------------------------------------------------- + +describe('pooled and unpooled emitters agree (differential)', () => { + const G256 = point(P256.gx, P256.gy, 32); + const GSEC = point(SECP.gx, SECP.gy, 32); + const G384 = point(P384.gx, P384.gy, 48); + + it('p256Add: G + G (the doubling path)', () => { + expectSame(emitP256Add, [push(G256), push(G256)]); + }); + + it('p256Negate: -G', () => { + expectSame(emitP256Negate, [push(G256)]); + }); + + it('p256OnCurve: accepts G', () => { + const { off } = expectSame(emitP256OnCurve, [push(G256)]); + expect(off.stack).toEqual(['01']); + }); + + it('p256OnCurve: rejects a point off the curve', () => { + const { off } = expectSame(emitP256OnCurve, [push(point(P256.gx, P256.gy + 1n, 32))]); + expect(off.stack).toEqual(['']); + }); + + it('p256OnCurve: rejects a non-canonical x >= p', () => { + // The pooled prime is what the canonicity guard compares against, so this + // is the case that would break first if the pool ever served a stale slot. + expectSame(emitP256OnCurve, [push(point(P256.gx + P256.p, P256.gy, 32))]); + }); + + it.each([1n, 2n, 3n, 7n, P256.n - 1n, 0n, P256.n])('p256Mul: G * %s', (k) => { + expectSame(emitP256Mul, [push(G256), pushN(k)]); + }); + + it('p384Add: G + G', () => { + expectSame(emitP384Add, [push(G384), push(G384)]); + }); + + it('p384Negate / p384OnCurve on G', () => { + expectSame(emitP384Negate, [push(G384)]); + expectSame(emitP384OnCurve, [push(G384)]); + }); + + it('ecAdd: G + G (secp256k1)', () => { + expectSame(emitEcAdd, [push(GSEC), push(GSEC)]); + }); + + it('ecNegate / ecOnCurve on G (secp256k1)', () => { + expectSame(emitEcNegate, [push(GSEC)]); + expectSame(emitEcOnCurve, [push(GSEC)]); + }); + + it.each([1n, 2n, 5n, SECP.n - 1n, 0n])('ecMul: G * %s (secp256k1)', (k) => { + expectSame(emitEcMul, [push(GSEC), pushN(k)]); + }); + + it('agrees on garbage inputs too — both must fail the same way', () => { + // Totality matters: these builtins are specified as "consume N, push 1" + // for ANY argument bytes, so a divergence in the ERROR is as bad as a + // divergence in the result. + expectSame(emitP256OnCurve, [push('00'.repeat(64))]); + expectSame(emitP256Add, [push('ff'.repeat(64)), push('00'.repeat(64))]); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Oracle — OpenSSL signatures, both variants +// --------------------------------------------------------------------------- + +/** DER SEQUENCE { INTEGER r, INTEGER s } -> fixed-width r||s. */ +function derToRaw(der: Buffer, bytes: number): string { + let i = 0; + if (der[i++] !== 0x30) throw new Error('not a DER sequence'); + if (der[i]! & 0x80) i += 1 + (der[i]! & 0x7f); else i += 1; + const readInt = (): bigint => { + if (der[i++] !== 0x02) throw new Error('not a DER integer'); + const len = der[i++]!; + const v = BigInt('0x' + der.subarray(i, i + len).toString('hex')); + i += len; + return v; + }; + const r = readInt(); + const s = readInt(); + const w = bytes * 2; + return r.toString(16).padStart(w, '0') + s.toString(16).padStart(w, '0'); +} + +const CURVES = [ + { name: 'p256', node: 'prime256v1' as const, bytes: 32, emit: emitVerifyECDSA_P256, n: P256.n }, + { name: 'p384', node: 'secp384r1' as const, bytes: 48, emit: emitVerifyECDSA_P384, + n: 0xffffffffffffffffffffffffffffffffffffffffffffffffc7634d81f4372ddf581a0db248b0a77aecec196accc52973n }, +]; + +for (const c of CURVES) { + describe(`${c.name} verifyECDSA agrees under pooling (OpenSSL oracle)`, () => { + const { privateKey, publicKey } = generateKeyPairSync('ec', { namedCurve: c.node }); + const pub = publicKey.export({ format: 'der', type: 'spki' }) as Buffer; + const uncompressed = pub.subarray(pub.length - (1 + c.bytes * 2)).toString('hex'); + const w = c.bytes * 2; + const qx = BigInt('0x' + uncompressed.slice(2, 2 + w)); + const qy = BigInt('0x' + uncompressed.slice(2 + w)); + const compressed = ((qy & 1n) === 0n ? '02' : '03') + hx(qx, c.bytes); + + const msgHex = '52c3ad6172206d657373616765'; // "Rúnar message" + const signer = createSign('sha256'); + signer.update(Buffer.from(msgHex, 'hex')); + const sigHex = derToRaw(signer.sign(privateKey) as Buffer, c.bytes); + + const verify = (msg: string, sig: string, pk: string, pooled: boolean): boolean => { + const r = run(c.emit, [push(msg), push(sig), push(pk)], pooled); + expect(r.error, 'verifier aborted instead of returning a boolean').toBe(null); + expect(r.stack.length, 'specified as 3 args in, 1 boolean out').toBe(1); + return r.stack[0] !== '' && r.stack[0] !== '00'; + }; + + const zero = '0'.repeat(w); + const rGen = sigHex.slice(0, w); + const sGen = sigHex.slice(w); + const flipped = (compressed.slice(0, 2) === '02' ? '03' : '02') + compressed.slice(2); + + const CASES: Array<[string, string, string, string, boolean]> = [ + ['genuine signature', msgHex, sigHex, compressed, true], + ['wrong message', msgHex + '00', sigHex, compressed, false], + ['wrong pubkey parity', msgHex, sigHex, flipped, false], + ['all-zero signature (universal forgery)', msgHex, zero + zero, compressed, false], + ['r = 0', msgHex, zero + sGen, compressed, false], + ['s = 0', msgHex, rGen + zero, compressed, false], + ['r = n', msgHex, hx(c.n, c.bytes) + sGen, compressed, false], + ['s = n', msgHex, rGen + hx(c.n, c.bytes), compressed, false], + ['truncated signature', msgHex, sigHex.slice(0, w), compressed, false], + ['oversized signature', msgHex, sigHex + 'ff', compressed, false], + ]; + + it.each(CASES)('%s', (_label, msg, sig, pk, want) => { + expect(verify(msg, sig, pk, false)).toBe(want); + expect(verify(msg, sig, pk, true)).toBe(want); + }); + + it('does not blow the interpreter stack budget', () => { + const off = run(c.emit, [push(msgHex), push(sigHex), push(compressed)], false); + const on = run(c.emit, [push(msgHex), push(sigHex), push(compressed)], true); + // The pool is a small constant number of extra resident slots. + expect(on.maxStackDepth).toBeLessThanOrEqual(off.maxStackDepth + 8); + expect(on.maxStackDepth).toBeLessThan(800); + }); + }); +} diff --git a/packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts b/packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts new file mode 100644 index 00000000..b6eea482 --- /dev/null +++ b/packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts @@ -0,0 +1,144 @@ +/** + * Liveness scheduler + EC constant pool — semantic equivalence, source vs script. + * + * Both experiments move values around the stack (the scheduler parks results + * on the alt stack; the pool serves a constant from a resident slot). That is + * exactly the class of change that can produce a script which still runs and + * still leaves a truthy top-of-stack while computing something else, so a byte + * count proves nothing on its own. + * + * `runDifferentialExecution` compiles a contract, executes the deployed script + * on the real @bsv/sdk engine, AND runs the same spend through the ANF + * interpreter — a source-semantics oracle that knows nothing about stack + * layout. Running it once per variant on the same witness gives translation + * validation: + * + * current(w) == variant(w) == interpreter(w) for every witness w + * + * The witnesses are NOT invented here: they come from `conformance/witnesses/`, + * the same specs CI already runs, so every expectation is one the repo has + * independently committed to. `conformance/witnesses/coverage-claims.test.ts` + * enforces that each spec carries at least one accept AND one reject, so a + * variant that made everything pass cannot slip through. + */ + +import { describe, it, expect } from 'vitest'; +import { readFileSync, readdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { runDifferentialExecution, type WitnessArg } from '../oracle/differential-execution.js'; + +const CONFORMANCE = resolve(__dirname, '../../../../conformance'); +const WITNESS_DIR = join(CONFORMANCE, 'witnesses'); +const TESTS_DIR = join(CONFORMANCE, 'tests'); + +/** Decode a method-argument literal: bigint ("27n"), boolean, or bytes ("0x…"). */ +function decodeArg(v: unknown): WitnessArg { + if (typeof v === 'boolean') return v; + if (typeof v === 'string') { + if (/^-?\d+n$/.test(v)) return BigInt(v.slice(0, -1)); + if (v.startsWith('0x')) return Uint8Array.from(Buffer.from(v.slice(2), 'hex')); + } + throw new Error(`unencodable witness arg: ${JSON.stringify(v)}`); +} + +function decodeCtor(v: unknown): bigint | boolean | string { + if (typeof v === 'boolean') return v; + if (typeof v === 'string') { + if (/^-?\d+n$/.test(v)) return BigInt(v.slice(0, -1)); + if (v.startsWith('0x')) return v.slice(2); + } + throw new Error(`unencodable constructor arg: ${JSON.stringify(v)}`); +} + +interface Spend { method: string; args: unknown[]; expect: 'accept' | 'reject'; note?: string } +interface Spec { fixture: string; constructorArgs?: Record; spends: Spend[] } + +const NON_SPEC_JSON = new Set(['coverage-ledger.json']); +const SPECS: Spec[] = readdirSync(WITNESS_DIR) + .filter(f => f.endsWith('.json') && !NON_SPEC_JSON.has(f)) + .sort() + .map(f => JSON.parse(readFileSync(join(WITNESS_DIR, f), 'utf-8')) as Spec); + +/** Variants under test. `current` is the baseline every other is compared to. */ +const VARIANTS = [ + { name: 'liveness', opts: { schedulerMode: 'liveness' as const } }, + { name: 'ec-pool', opts: { ecConstantPool: true } }, + { name: 'both', opts: { schedulerMode: 'liveness' as const, ecConstantPool: true } }, +]; + +describe('experimental backends preserve acceptance', () => { + it('found the witness corpus', () => { + expect(SPECS.length).toBeGreaterThanOrEqual(10); + }); + + for (const spec of SPECS) { + const fixtureDir = join(TESTS_DIR, spec.fixture); + const srcCfg = JSON.parse(readFileSync(join(fixtureDir, 'source.json'), 'utf-8')) as + { sources?: Record; path?: string }; + const tsRel = srcCfg.sources?.['.runar.ts'] ?? srcCfg.path; + if (!tsRel) throw new Error(`no .runar.ts source in ${spec.fixture}/source.json`); + const srcPath = resolve(fixtureDir, tsRel); + const source = readFileSync(srcPath, 'utf-8'); + const fileName = srcPath.split('/').pop()!; + const ctor: Record = {}; + for (const [k, v] of Object.entries(spec.constructorArgs ?? {})) ctor[k] = decodeCtor(v); + + describe(spec.fixture, () => { + for (const s of spec.spends) { + for (const variant of VARIANTS) { + it(`${variant.name}: ${s.method}(${s.args.join(',')}) → ${s.expect}`, () => { + const common = { + source, fileName, method: s.method, + args: s.args.map(decodeArg), constructorArgs: ctor, + }; + const base = runDifferentialExecution(common); + const other = runDifferentialExecution({ ...common, ...variant.opts }); + + // The witness spec itself must hold, or the comparison is vacuous. + expect(base.vmAccepted, 'witness spec disagrees with the shipping compiler') + .toBe(s.expect === 'accept'); + // Translation validation, both directions against the interpreter. + expect(other.vmAccepted).toBe(base.vmAccepted); + expect(base.vmAccepted).toBe(base.interpreterAccepted); + expect(other.vmAccepted).toBe(other.interpreterAccepted); + expect(other.vmError ?? null).toBe(base.vmError ?? null); + }); + } + } + }); + } + + it('the liveness scheduler really does change bytes somewhere in this corpus', () => { + // Guards against the whole suite passing because every variant compiled to + // the identical script. + const changed: string[] = []; + for (const spec of SPECS) { + const fixtureDir = join(TESTS_DIR, spec.fixture); + const srcCfg = JSON.parse(readFileSync(join(fixtureDir, 'source.json'), 'utf-8')) as + { sources?: Record; path?: string }; + const tsRel = srcCfg.sources?.['.runar.ts'] ?? srcCfg.path; + if (!tsRel) continue; + const srcPath = resolve(fixtureDir, tsRel); + const ctor: Record = {}; + for (const [k, v] of Object.entries(spec.constructorArgs ?? {})) ctor[k] = decodeCtor(v); + const first = spec.spends[0]!; + const common = { + source: readFileSync(srcPath, 'utf-8'), + fileName: srcPath.split('/').pop()!, + method: first.method, + args: first.args.map(decodeArg), + constructorArgs: ctor, + }; + const base = runDifferentialExecution(common); + const sched = runDifferentialExecution({ ...common, schedulerMode: 'liveness' }); + if (sched.lockingHex !== base.lockingHex) { + changed.push(spec.fixture); + // And it must never be bigger — the cost model picks per method. + expect(sched.lockingHex.length, `${spec.fixture} grew`) + .toBeLessThan(base.lockingHex.length); + } + } + expect(changed.length, 'scheduler was a no-op on every witnessed fixture').toBeGreaterThan(0); + console.log(` scheduler changed bytes on: ${changed.join(', ')}`); + }); +}); diff --git a/packages/runar-testing/src/__tests__/scheduler-headroom.test.ts b/packages/runar-testing/src/__tests__/scheduler-headroom.test.ts new file mode 100644 index 00000000..f8e4c2e2 --- /dev/null +++ b/packages/runar-testing/src/__tests__/scheduler-headroom.test.ts @@ -0,0 +1,127 @@ +/** + * Headroom probe for the stack scheduler. + * + * `conformance/tests/arithmetic` is the smallest fixture whose bytes are + * produced ENTIRELY by the generic ANF -> Stack lowering: no crypto macro, no + * sighash scaffolding, no state continuation. 16 of its 28 bytes (57 %) are + * stack access. That makes it the honest measuring stick for "how much can a + * better schedule win on ordinary contracts?". + * + * This test pins two things: + * + * 1. what the compiler emits today, and + * 2. that a hand-written alternative schedule — operands held hot at the top, + * finished results parked on the alt stack — accepts and rejects exactly + * the same inputs while being materially smaller. + * + * (2) is not a claim about what the compiler does; it is the TARGET the + * liveness scheduler is aimed at, executed on the real interpreter so the + * headroom number in `docs/experiments/stack-scheduler-design.md` is measured + * rather than estimated. If a future scheduler beats it, tighten this test. + */ + +import { describe, it, expect } from 'vitest'; +import { ScriptVM } from '../vm/script-vm.js'; + +/** Encode a bigint as a minimally-encoded Bitcoin script number push. */ +function pushNum(n: bigint): string { + if (n === 0n) return '00'; + if (n >= 1n && n <= 16n) return (0x50 + Number(n)).toString(16).padStart(2, '0'); + const neg = n < 0n; + let v = neg ? -n : n; + const bytes: number[] = []; + while (v > 0n) { bytes.push(Number(v & 0xffn)); v >>= 8n; } + if (bytes[bytes.length - 1]! & 0x80) bytes.push(neg ? 0x80 : 0x00); + else if (neg) bytes[bytes.length - 1] = bytes[bytes.length - 1]! | 0x80; + return bytes.length.toString(16).padStart(2, '0') + bytes.map(b => b.toString(16).padStart(2, '0')).join(''); +} + +/** + * What the compiler emits today for `Arithmetic.verify`, with the constructor + * placeholder (`00`) replaced by a real target push. + * + * OP_2DUP OP_ADD sum + * OP_2 OP_PICK OP_2 OP_PICK OP_SUB diff + * OP_3 OP_PICK OP_3 OP_PICK OP_MUL prod + * OP_4 OP_ROLL OP_4 OP_ROLL OP_DIV quot + * OP_3 OP_ROLL OP_3 OP_ROLL OP_ADD OP_ROT OP_ADD OP_SWAP OP_ADD + * OP_NUMEQUAL + */ +function currentSchedule(target: bigint): string { + return `6e9352795279945379537995547a547a96537a537a937b937c93${pushNum(target)}9c`; +} + +/** + * The same computation, scheduled so `a` and `b` never leave the top two + * slots and each finished result is spilled to the alt stack: + * + * OP_2DUP OP_ADD OP_TOALTSTACK sum -> alt + * OP_2DUP OP_SUB OP_TOALTSTACK diff -> alt + * OP_2DUP OP_MUL OP_TOALTSTACK prod -> alt + * OP_DIV quot (consumes a, b) + * OP_FROMALTSTACK OP_ADD + prod + * OP_FROMALTSTACK OP_ADD + diff + * OP_FROMALTSTACK OP_ADD + sum + * OP_NUMEQUAL + * + * Addition is associative and commutative over script numbers here, so the + * reversed accumulation order is value-identical. + */ +function altStackSchedule(target: bigint): string { + return `6e936b6e946b6e956b966c936c936c93${pushNum(target)}9c`; +} + +function run(scriptHex: string, a: bigint, b: bigint): boolean { + const vm = new ScriptVM(); + const unlocking = `${pushNum(a)}${pushNum(b)}`; + const r = vm.execute( + Uint8Array.from(Buffer.from(unlocking, 'hex')), + Uint8Array.from(Buffer.from(scriptHex, 'hex')), + ); + return r.success; +} + +/** a + b, a - b, a * b, a / b summed — the contract's `result`. */ +function expected(a: bigint, b: bigint): bigint { + // Script's OP_DIV truncates toward zero, which matches bigint division. + return (a + b) + (a - b) + a * b + a / b; +} + +const CASES: [bigint, bigint][] = [ + [7n, 3n], [3n, 7n], [1n, 1n], [100n, 7n], [-5n, 3n], [5n, -3n], + [-5n, -3n], [0n, 1n], [16n, 16n], [17n, 2n], [255n, 4n], [-1n, -1n], + [1000n, 3n], [2n, 1000n], +]; + +describe('stack scheduler headroom (conformance/tests/arithmetic)', () => { + it('pins the byte cost of both schedules', () => { + // 5 is the byte cost of the `target` push in these probes (4-byte push of + // a value that needs a sign byte); both schedules carry the same one, so + // the difference is entirely scheduling. + const t = 1000n; + const cur = currentSchedule(t).length / 2; + const alt = altStackSchedule(t).length / 2; + expect(cur).toBe(30); + expect(alt).toBe(20); + // 33 % fewer bytes, all of it stack traffic. + expect(1 - alt / cur).toBeGreaterThan(0.3); + }); + + it('the emitted schedule matches the checked-in golden modulo the placeholder', () => { + // Golden is the template: `00` where the constructor arg is spliced in. + const template = '6e9352795279945379537995547a547a96537a537a937b937c93009c'; + expect(currentSchedule(0n)).toBe(template); + }); + + it.each(CASES)('both schedules accept exactly the right target for a=%s b=%s', (a, b) => { + const want = expected(a, b); + expect(run(currentSchedule(want), a, b)).toBe(true); + expect(run(altStackSchedule(want), a, b)).toBe(true); + }); + + it.each(CASES)('both schedules reject a wrong target for a=%s b=%s', (a, b) => { + const wrong = expected(a, b) + 1n; + expect(run(currentSchedule(wrong), a, b)).toBe(false); + expect(run(altStackSchedule(wrong), a, b)).toBe(false); + }); +}); diff --git a/packages/runar-testing/src/oracle/differential-execution.ts b/packages/runar-testing/src/oracle/differential-execution.ts index 8c68f3ed..b40becc5 100644 --- a/packages/runar-testing/src/oracle/differential-execution.ts +++ b/packages/runar-testing/src/oracle/differential-execution.ts @@ -50,6 +50,14 @@ export interface DiffExecOptions { args: WitnessArg[]; // method arguments (interpreter + witness order) constructorArgs?: Record; disableConstantFolding?: boolean; // default false → fold-ON deployed bytes + /** + * EXPERIMENTAL backend toggles. Present so a size experiment can be run + * through this same source-vs-script oracle: compile the SAME contract with + * a flag off and on, execute both on the SAME witness, and require identical + * acceptance. Both default to the shipping path. + */ + schedulerMode?: 'current' | 'liveness'; + ecConstantPool?: boolean; } export interface DiffExecResult { @@ -88,6 +96,8 @@ export function runDifferentialExecution(opts: DiffExecOptions): DiffExecResult fileName: opts.fileName, disableConstantFolding: opts.disableConstantFolding ?? false, constructorArgs: ctor, + schedulerMode: opts.schedulerMode, + ecConstantPool: opts.ecConstantPool, }); if (!compiled.success || !compiled.artifact) { const errs = compiled.diagnostics From 7e3e1a1ea041aa9f0bae9060f2e2be657451964c Mon Sep 17 00:00:00 2001 From: Siggi Date: Fri, 28 Aug 2026 18:15:36 +0200 Subject: [PATCH 3/4] docs(experiments): record the Phase 0-2 script-size findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three reports under docs/experiments/, all reproducible with `pnpm --filter runar-conformance run script-metrics`. script-size-optimization-baseline.md 58% of every byte the compiler has ever emitted is a constant push, and 56% of the whole 13.5 MB corpus is nine numbers — each curve's field prime — pushed over and over. p256-wallet is 72.7% constant pushes, 71.0% of the fixture in a single 33-byte literal repeated 20,025 times. Also records that brief Phase 3 (fix-point peephole) and Phase 15 (OP_PUSH_TX binding) already ship. stack-scheduler-design.md The current lowering algorithm with line references, the measured inefficiencies, the byte-cost function, the correctness invariants, and the benchmark plan — written before the prototype, then updated with what actually happened, including the miscompile the witness corpus caught and why a passing bisect can be vacuous. script-size-optimizer-results.md What is generic, what is not, and where the remaining bytes are. After pooling, p256-wallet's 304,463 bytes are 70.6% stack traffic and 27.2% arithmetic — all of it the modular-reduction sequence itself, ~20,000 times. Reaching ~30 kB needs the algebra, not more scheduling: modular-domain analysis first, then Straus/Shamir, then a fixed-base comb. Two plan assumptions died on measurement and are recorded as dead rather than quietly dropped: - Eager dead-slot retirement. 657 of 387,749 pick/roll sites in the corpus are deeper than 16; typical depths are 2-5, and depths 0-2 are single-byte opcodes. Every drop would cost bytes to save nothing. - That a generic scheduler could reach P-256 at all. The crypto emitters build their own stack layout through ECTracker and never pass through 05-stack-lower.ts — 13.4 MB of the 13.5 MB corpus is out of its reach. --- .../script-size-optimization-baseline.md | 305 ++++++++++++++ .../script-size-optimizer-results.md | 288 +++++++++++++ docs/experiments/stack-scheduler-design.md | 393 ++++++++++++++++++ 3 files changed, 986 insertions(+) create mode 100644 docs/experiments/script-size-optimization-baseline.md create mode 100644 docs/experiments/script-size-optimizer-results.md create mode 100644 docs/experiments/stack-scheduler-design.md diff --git a/docs/experiments/script-size-optimization-baseline.md b/docs/experiments/script-size-optimization-baseline.md new file mode 100644 index 00000000..e82e3956 --- /dev/null +++ b/docs/experiments/script-size-optimization-baseline.md @@ -0,0 +1,305 @@ +# Script-size optimization — Phase 0 baseline + +**Date:** 2026-08-28 · **Compiler:** TypeScript reference tier, default options (constant folding ON, EC optimizer ON, peephole ON) +**Source of truth:** the checked-in `conformance/tests//expected-script.hex` goldens, 72 fixtures, 13,526,545 bytes total. + +Reproduce every number here with: + +```bash +pnpm --filter runar-conformance run script-metrics # summary table +pnpm --filter runar-conformance run script-metrics -- --fixture p256-wallet --detail +pnpm --filter runar-conformance run script-metrics -- --json out.json # machine-readable +``` + +Instrumentation is read-only and does not change compilation output: +`packages/runar-compiler/src/metrics/script-metrics.ts` (byte classifier), +`packages/runar-compiler/src/metrics/cost-model.ts` (`estimateScriptBytes`, asserted +byte-exact against `06-emit.ts` over the whole corpus in `__tests__/cost-model.test.ts`), +`conformance/runner/script-metrics.ts` (the runner). + +--- + +## 1. The headline + +**58 % of every script byte Rúnar has ever emitted is a constant push, and 56 % of the +entire corpus is nine distinct constants pushed over and over.** + +| category | bytes | share of corpus | +|---|---:|---:| +| const-push | 7,840,690 | **58.0 %** | +| stack-shuffle | 3,133,223 | 23.2 % | +| arithmetic | 1,047,644 | 7.7 % | +| bytes (CAT/SPLIT/SIZE/EQUAL) | 878,631 | 6.5 % | +| small-int-push | 392,378 | 2.9 % | +| control | 160,000 | 1.2 % | +| crypto | 73,979 | 0.5 % | + +The nine largest single constants, all of them a curve's field prime `p` (or group order `n`): + +| fixture | constant | push size | pushes | bytes | share of that fixture | +|---|---|---:|---:|---:|---:| +| p384-wallet | P-384 `p` | 49 B | 30,577 | 1,528,850 | 77.9 % | +| p384-primitives | P-384 `p` | 49 B | 29,925 | 1,496,250 | 79.4 % | +| ec-primitives | secp256k1 `p` | 33 B | 28,102 | 955,468 | 71.7 % | +| ec-demo | secp256k1 `p` | 33 B | 28,102 | 955,468 | 71.7 % | +| **p256-wallet** | **P-256 `p`** | **33 B** | **20,025** | **680,850** | **71.0 %** | +| p256-primitives | P-256 `p` | 33 B | 19,755 | 671,670 | 72.4 % | +| schnorr-zkp | secp256k1 `p` | 33 B | 18,551 | 630,734 | 72.1 % | +| ec-unit | secp256k1 `p` | 33 B | 10,092 | 343,128 | 71.5 % | +| convergence-proof | secp256k1 `p` | 33 B | 9,547 | 324,598 | 71.8 % | + +Total: **7,587,016 bytes — 56 % of the corpus — spent re-pushing nine numbers.** + +### Why + +`cFieldMod` / `fieldMod` push the prime inline at *every* modular reduction: + +```ts +// packages/runar-compiler/src/passes/p256-p384-codegen.ts:135 +function cFieldMod(t: ECTracker, aName: string, resultName: string, c: CurveParams): void { + t.toTop(aName); + pushFieldP(t, '_fmod_p', c); // <-- 34 bytes (P-256) / 50 bytes (P-384), every time + t.rawBlock([aName, '_fmod_p'], resultName, (e) => { + e({ op: 'opcode', code: 'OP_2DUP' }); e({ op: 'opcode', code: 'OP_MOD' }); + e({ op: 'rot' }); e({ op: 'drop' }); e({ op: 'over' }); + e({ op: 'opcode', code: 'OP_ADD' }); e({ op: 'swap' }); e({ op: 'opcode', code: 'OP_MOD' }); + }); +} +``` + +Every `cFieldAdd` / `cFieldSub` / `cFieldMul` / `cFieldSqr` / `cFieldMulConst` ends in one of +these. `cEmitMul` unrolls 257 (P-256) / 385 (P-384) double-and-add rounds, `cFieldInv` and +`cGroupInv` unroll full Fermat ladders — so the prime push is multiplied by the unroll factor. + +A prime kept in a stack slot and copied with `push(depth); OP_PICK` costs **2–3 bytes** +instead of 34 or 50. Break-even is at two uses. + +--- + +## 2. p256-wallet — the brief's 959 kB reference, in detail + +`conformance/tests/p256-wallet` is **958,792 bytes**, which is the "959,592 B baseline +reference implementation" the optimization brief targets. It is a hybrid secp256k1 + P-256 +wallet: a P2PKH gate, then `verifyECDSA_P256(sig, p256Sig, p256PubKey)`. + +| category | bytes | share | +|---|---:|---:| +| const-push | 697,019 | 72.7 % | +| stack-shuffle | 173,967 | 18.1 % | +| arithmetic | 82,863 | 8.6 % | +| bytes | 1,210 | 0.1 % | +| small-int-push | 2,698 | 0.3 % | +| control | 1,031 | 0.1 % | +| crypto | 4 | 0.0 % | + +| repeated constant | size | pushes | bytes | share | +|---|---:|---:|---:|---:| +| P-256 field prime `p` | 33 B | 20,025 | 680,850 | 71.0 % | +| P-256 group order `n` | 33 B | 430 | 14,620 | 1.5 % | + +Top opcodes: `OP_MOD`×41,418 · `OP_ROT`×30,406 · `OP_SWAP`×27,342 · `OP_OVER`×23,417 · +`OP_DROP`×22,558 · `OP_ADD`×20,978 · `OP_2DUP`×20,453 · `OP_MUL`×13,214. + +Note the shape: **41,418 `OP_MOD` against 20,025 prime pushes** — two `OP_MOD` per reduction. +That is the sign-normalisation tail (`2DUP MOD ROT DROP OVER ADD SWAP MOD`), which exists +because `OP_MOD` takes the sign of the dividend. For a product of two values already reduced +into `[0, p)` the dividend is non-negative and the tail is dead weight: 6 of the 8 opcodes, +plus the second prime reference. That is a modular-domain-analysis win (brief Phase 4/5), not +a scheduling one. + +### Where the arithmetic actually goes + +Op-count goldens (`packages/runar-compiler/src/__tests__/p256-p384-codegen.test.ts:111`): + +| emitter | ops | measured bytes | +|---|---:|---:| +| `emitVerifyECDSA_P256` | 297,331 | 974,024 | +| `emitP256Mul` / `emitP256MulGen` | 140,036 / 140,038 | 459,746 / 459,812 | +| `emitP256Add` | 6,663 | 19,906 | +| `emitVerifyECDSA_P384` | 453,307 | 1,987,394 | +| `emitP384Mul` | 211,178 | 927,350 | + +`cEmitVerifyECDSA` runs **two independent 257-round ladders** (`u1·G` at `:1412`, `u2·Q` at +`:1442`) plus **three unrolled Fermat exponentiations** (`cFieldInv` 382 field muls, +`cGroupInv` 423, `cFieldPow` for the decompression sqrt 286). Nothing is shared between the +two ladders and no point is precomputed, even though `G` is a compile-time constant. + +--- + +## 3. Two populations, two different bottlenecks + +The corpus splits cleanly, and the split decides which optimization can touch which fixture. + +### EC / field-arithmetic fixtures — dominated by constants (72–80 % const-push) + +`p256-*`, `p384-*`, `ec-*`, `schnorr-zkp`, `convergence-proof`, `babybear*`. These scripts are +emitted by hand-written macro modules (`ec-codegen.ts`, `p256-p384-codegen.ts`, +`babybear-codegen.ts`, …) that build their own stack layout through `ECTracker` and its +clones. **They never pass through `05-stack-lower.ts`.** A generic ANF→Stack scheduler cannot +move a single byte of them. + +### Ordinary contracts — dominated by stack traffic (35–68 % stack-shuffle) + +Everything from `stateful-counter` (1,875 B) up through `math-demo` (17,348 B), and the small +fixtures most of all: `arithmetic` 67.9 %, `bounded-loop` 57.1 %, `multisig` 58.8 %, +`if-without-else-multi-temp` 55.3 %. These *are* produced by `05-stack-lower.ts`, and the +~30 % const-push in the mid-size stateful fixtures is largely BIP-143 sighash scaffolding, +not user data. + +### Hash / post-quantum fixtures — dominated by byte plumbing + +SLH-DSA (`OP_CAT`×80,120, `OP_SPLIT`×50,221 in the 128f fixture), SHA-256 and BLAKE3 sit at +2–17 % const-push, 36–44 % stack-shuffle, 23–28 % `bytes`. Constant pooling buys them almost +nothing; scheduling and byte-op fusion are the levers. + +| population | fixtures | const-push | stack-shuffle | reachable by | +|---|---|---:|---:|---| +| EC / field arithmetic | 9 (10.2 MB) | 72–80 % | 13–19 % | codegen-level constant pooling | +| hash / post-quantum | 12 (3.0 MB) | 2–17 % | 36–44 % | scheduling, byte-op fusion | +| ordinary contracts | 51 (0.1 MB) | 0–33 % | 35–68 % | generic liveness scheduler | + +--- + +## 4. Ranked byte sinks + +1. **Repeated field-prime pushes — 7,587,016 B (56 % of the corpus).** One pooled stack slot + per curve constant. Codegen-level (`ECTracker`), not a generic pass; the *policy* + (pool when `n_uses × push_cost > pool_cost + n_uses × pick_cost`) is generic and belongs + in the cost model. +2. **Stack traffic — 3,133,223 B (23 %).** Split roughly evenly between the EC macros' + `ECTracker.toTop`/`copyToTop` churn and `05-stack-lower.ts`'s `bringToTop`. The generic + half is addressable by liveness-driven scheduling; see + [`stack-scheduler-design.md`](stack-scheduler-design.md). +3. **The redundant second `OP_MOD` — ~20,000 reductions per EC fixture × 6 opcodes.** + Requires knowing an operand is already reduced (modular-domain analysis, brief Phase 4). +4. **Unrolled Fermat inversion.** 382 / 423 / 286 field muls per P-256 verify, three times. + An addition chain cuts each by ~30 %; a witness-supplied inverse (brief Phase 7) removes + them almost entirely. +5. **Two independent scalar ladders.** Straus/Shamir halves the doubling work; a fixed-base + comb for `u1·G` removes it (brief Phases 9–11). +6. **`emitReverse32` / `emitReverse48`.** 7 ops × 32 (or 48) per byte-order reversal, called + on every point decompose/compose. + +--- + +## 5. What Phase 0 already settles about the brief + +- **Phase 3 (fix-point peephole) is already done.** `optimizeStackIR` + (`optimizer/peephole.ts:507`) iterates `applyOnePass` to a fixed point with a 100-iteration + cap and recurses into `if` arms first. 28 rules, mirrored declaratively in + `optimizer/peephole-rules.ts` and executed pattern-vs-replacement through the `ScriptVM` by + `__tests__/peephole-exhaustive.test.ts`. What remains for Phase 3 is *more rules*, not a + fix-point driver. +- **Phase 15 (OP_PUSH_TX / CODESEPARATOR transaction binding) already ships** as + `passes/oppushtx-codegen.ts` — the BUG-100 fix derives the ECDSA signature from the pushed + preimage on-chain, so `OP_CHECKSIG` passes only when `hash256(preimage)` is the real + sighash. +- **Phase 2's generic scheduler cannot reach P-256.** See §3. The two must be prototyped + separately or the P-256 number will not move at all. +- **Phase 1's cost model is exact.** `estimateScriptBytes` agrees with `emitMethod` to the + byte on every method of all 67 fixtures that ship a `.runar.ts`, before and after peephole. + +--- + +## 6. Full corpus table + +Byte category shares per fixture, largest script first. + +| fixture | bytes | ops | const-push | stack-shuffle | arithmetic | bytes | crypto | small-int-push | control | other | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| p384-wallet | 1963300 | 430181 | 1565148 (79.7%) | 264286 (13.5%) | 126571 (6.4%) | 1754 (0.1%) | 4 (0.0%) | 3994 (0.2%) | 1543 (0.1%) | — | +| p384-primitives | 1883767 | 415719 | 1498750 (79.6%) | 256133 (13.6%) | 121250 (6.4%) | 1976 (0.1%) | — | 4109 (0.2%) | 1549 (0.1%) | — | +| ec-demo | 1332782 | 403869 | 957857 (71.9%) | 249884 (18.7%) | 113993 (8.6%) | 4208 (0.3%) | — | 5256 (0.4%) | 1584 (0.1%) | — | +| ec-primitives | 1332782 | 403869 | 957857 (71.9%) | 249884 (18.7%) | 113993 (8.6%) | 4208 (0.3%) | — | 5256 (0.4%) | 1584 (0.1%) | — | +| p256-wallet | 958792 | 282747 | 697019 (72.7%) | 173967 (18.1%) | 82863 (8.6%) | 1210 (0.1%) | 4 (0.0%) | 2698 (0.3%) | 1031 (0.1%) | — | +| p256-primitives | 928219 | 275241 | 673254 (72.5%) | 169769 (18.3%) | 80058 (8.6%) | 1336 (0.1%) | — | 2765 (0.3%) | 1037 (0.1%) | — | +| schnorr-zkp | 875189 | 262004 | 632257 (72.2%) | 162557 (18.6%) | 75250 (8.6%) | 1335 (0.2%) | 1 (0.0%) | 2759 (0.3%) | 1030 (0.1%) | — | +| post-quantum-slhdsa-192f | 788039 | 757325 | 54761 (6.9%) | 339659 (43.1%) | 56908 (7.2%) | 206316 (26.2%) | 17549 (2.2%) | 78196 (9.9%) | 34650 (4.4%) | — | +| post-quantum-slhdsa-256f | 729363 | 716143 | 19626 (2.7%) | 327166 (44.9%) | 57832 (7.9%) | 191936 (26.3%) | 17837 (2.4%) | 79752 (10.9%) | 35214 (4.8%) | — | +| post-quantum-slhdsa-128f | 533911 | 525388 | 12149 (2.3%) | 235416 (44.1%) | 39293 (7.4%) | 143273 (26.8%) | 12137 (2.3%) | 67751 (12.7%) | 23892 (4.5%) | — | +| ec-unit | 479716 | 146119 | 343994 (71.7%) | 89666 (18.7%) | 40878 (8.5%) | 2407 (0.5%) | — | 2248 (0.5%) | 523 (0.1%) | — | +| convergence-proof | 452386 | 136798 | 325416 (71.9%) | 84502 (18.7%) | 38705 (8.6%) | 1467 (0.3%) | — | 1780 (0.4%) | 516 (0.1%) | — | +| post-quantum-slhdsa-256s | 369173 | 358620 | 15558 (4.2%) | 162192 (43.9%) | 28402 (7.7%) | 96981 (26.3%) | 8818 (2.4%) | 40026 (10.8%) | 17196 (4.7%) | — | +| post-quantum-slhdsa-192s | 276583 | 262513 | 23837 (8.6%) | 116220 (42.0%) | 19195 (6.9%) | 72337 (26.2%) | 5985 (2.2%) | 27396 (9.9%) | 11613 (4.2%) | — | +| sphincs-wallet | 188609 | 183128 | 7747 (4.1%) | 80932 (42.9%) | 13282 (7.0%) | 50562 (26.8%) | 4164 (2.2%) | 23879 (12.7%) | 8043 (4.3%) | — | +| post-quantum-slhdsa | 188597 | 183116 | 7747 (4.1%) | 80926 (42.9%) | 13282 (7.0%) | 50560 (26.8%) | 4161 (2.2%) | 23878 (12.7%) | 8043 (4.3%) | — | +| sha256-finalize | 69507 | 63949 | 9898 (14.2%) | 24810 (35.7%) | 10155 (14.6%) | 16262 (23.4%) | — | 8379 (12.1%) | 3 (0.0%) | — | +| sha256-compress | 23145 | 21294 | 3296 (14.2%) | 8262 (35.7%) | 3384 (14.6%) | 5413 (23.4%) | — | 2790 (12.1%) | — | — | +| blake3 | 22428 | 20773 | 2798 (12.5%) | 8742 (39.0%) | 1811 (8.1%) | 6336 (28.3%) | — | 2738 (12.2%) | 3 (0.0%) | — | +| stateful-wots-gate | 20519 | 18177 | 3443 (16.8%) | 8025 (39.1%) | 2237 (10.9%) | 2454 (12.0%) | 1011 (4.9%) | 320 (1.6%) | 3029 (14.8%) | — | +| post-quantum-wallet | 19594 | 17514 | 3154 (16.1%) | 7690 (39.2%) | 2213 (11.3%) | 2275 (11.6%) | 1010 (5.2%) | 237 (1.2%) | 3015 (15.4%) | — | +| post-quantum-wots | 19582 | 17502 | 3154 (16.1%) | 7684 (39.2%) | 2213 (11.3%) | 2273 (11.6%) | 1007 (5.1%) | 236 (1.2%) | 3015 (15.4%) | — | +| math-demo | 17348 | 13183 | 4591 (26.5%) | 6188 (35.7%) | 1212 (7.0%) | 2842 (16.4%) | 62 (0.4%) | 1513 (8.7%) | 940 (5.4%) | — | +| babybear-ext4 | 5471 | 3084 | 2973 (54.3%) | 1320 (24.1%) | 1141 (20.9%) | — | — | 31 (0.6%) | 6 (0.1%) | — | +| function-patterns | 3844 | 2794 | 1159 (30.2%) | 1415 (36.8%) | 111 (2.9%) | 724 (18.8%) | 20 (0.5%) | 350 (9.1%) | 65 (1.7%) | — | +| token-ft | 3154 | 2330 | 929 (29.5%) | 1139 (36.1%) | 89 (2.8%) | 612 (19.4%) | 16 (0.5%) | 294 (9.3%) | 75 (2.4%) | — | +| merge-locals-shapes | 3031 | 2236 | 882 (29.1%) | 1145 (37.8%) | 88 (2.9%) | 567 (18.7%) | 12 (0.4%) | 280 (9.2%) | 57 (1.9%) | — | +| private-helper-outputs | 2879 | 2080 | 886 (30.8%) | 1018 (35.4%) | 76 (2.6%) | 559 (19.4%) | 12 (0.4%) | 271 (9.4%) | 57 (2.0%) | — | +| assert-false-guard | 2033 | 1507 | 582 (28.6%) | 778 (38.3%) | 54 (2.7%) | 378 (18.6%) | 8 (0.4%) | 186 (9.1%) | 47 (2.3%) | — | +| loop-if-merged-locals | 2011 | 1481 | 588 (29.2%) | 750 (37.3%) | 60 (3.0%) | 378 (18.8%) | 8 (0.4%) | 186 (9.2%) | 41 (2.0%) | — | +| terminal-varlen-read | 1940 | 1415 | 584 (30.1%) | 688 (35.5%) | 55 (2.8%) | 375 (19.3%) | 7 (0.4%) | 178 (9.2%) | 53 (2.7%) | — | +| property-initializers | 1878 | 1354 | 578 (30.8%) | 674 (35.9%) | 50 (2.7%) | 362 (19.3%) | 8 (0.4%) | 175 (9.3%) | 31 (1.7%) | — | +| stateful | 1876 | 1352 | 578 (30.8%) | 674 (35.9%) | 50 (2.7%) | 362 (19.3%) | 8 (0.4%) | 174 (9.3%) | 30 (1.6%) | — | +| stateful-counter | 1875 | 1351 | 578 (30.8%) | 673 (35.9%) | 51 (2.7%) | 362 (19.3%) | 8 (0.4%) | 173 (9.2%) | 30 (1.6%) | — | +| stateful-bytestring | 1851 | 1335 | 569 (30.7%) | 660 (35.7%) | 48 (2.6%) | 357 (19.3%) | 8 (0.4%) | 171 (9.2%) | 38 (2.1%) | — | +| auction | 1794 | 1288 | 553 (30.8%) | 656 (36.6%) | 47 (2.6%) | 346 (19.3%) | 9 (0.5%) | 164 (9.1%) | 19 (1.1%) | — | +| token-nft | 1738 | 1234 | 549 (31.6%) | 630 (36.2%) | 43 (2.5%) | 332 (19.1%) | 9 (0.5%) | 157 (9.0%) | 18 (1.0%) | — | +| state-covenant | 1196 | 912 | 326 (27.3%) | 451 (37.7%) | 41 (3.4%) | 221 (18.5%) | 9 (0.8%) | 105 (8.8%) | 43 (3.6%) | — | +| branched-readonly-len | 1096 | 816 | 319 (29.1%) | 392 (35.8%) | 33 (3.0%) | 211 (19.3%) | 4 (0.4%) | 100 (9.1%) | 37 (3.4%) | — | +| conditional-data-output-stateful | 1015 | 740 | 308 (30.3%) | 356 (35.1%) | 27 (2.7%) | 197 (19.4%) | 4 (0.4%) | 97 (9.6%) | 26 (2.6%) | — | +| merge-locals-prop-updates | 1006 | 741 | 294 (29.2%) | 387 (38.5%) | 27 (2.7%) | 189 (18.8%) | 4 (0.4%) | 89 (8.8%) | 16 (1.6%) | — | +| add-raw-output | 1005 | 728 | 311 (30.9%) | 349 (34.7%) | 27 (2.7%) | 197 (19.6%) | 4 (0.4%) | 95 (9.5%) | 22 (2.2%) | — | +| add-data-output | 1004 | 729 | 308 (30.7%) | 351 (35.0%) | 27 (2.7%) | 197 (19.6%) | 4 (0.4%) | 95 (9.5%) | 22 (2.2%) | — | +| selector | 985 | 723 | 289 (29.3%) | 371 (37.7%) | 28 (2.8%) | 185 (18.8%) | 4 (0.4%) | 88 (8.9%) | 20 (2.0%) | — | +| branch-merged-locals | 963 | 699 | 292 (30.3%) | 354 (36.8%) | 24 (2.5%) | 185 (19.2%) | 4 (0.4%) | 88 (9.1%) | 16 (1.7%) | — | +| cond-write-multi-field | 957 | 693 | 292 (30.5%) | 345 (36.1%) | 26 (2.7%) | 185 (19.3%) | 4 (0.4%) | 89 (9.3%) | 16 (1.7%) | — | +| state-bigint-edges | 952 | 688 | 292 (30.7%) | 346 (36.3%) | 25 (2.6%) | 185 (19.4%) | 4 (0.4%) | 87 (9.1%) | 13 (1.4%) | — | +| intent-current-block-height | 944 | 682 | 289 (30.6%) | 338 (35.8%) | 26 (2.8%) | 185 (19.6%) | 4 (0.4%) | 88 (9.3%) | 14 (1.5%) | — | +| intent-prev-output-script | 942 | 680 | 289 (30.7%) | 339 (36.0%) | 25 (2.7%) | 183 (19.4%) | 5 (0.5%) | 87 (9.2%) | 14 (1.5%) | — | +| oversize-bigint-shift | 940 | 671 | 297 (31.6%) | 334 (35.5%) | 25 (2.7%) | 181 (19.3%) | 4 (0.4%) | 86 (9.1%) | 13 (1.4%) | — | +| state-ripemd160 | 931 | 668 | 291 (31.3%) | 336 (36.1%) | 23 (2.5%) | 179 (19.2%) | 4 (0.4%) | 84 (9.0%) | 14 (1.5%) | — | +| intent-output-p2pkh | 843 | 594 | 270 (32.0%) | 309 (36.7%) | 18 (2.1%) | 165 (19.6%) | 4 (0.5%) | 76 (9.0%) | 1 (0.1%) | — | +| covenant-vault | 795 | 550 | 262 (33.0%) | 290 (36.5%) | 13 (1.6%) | 151 (19.0%) | 5 (0.6%) | 73 (9.2%) | 1 (0.1%) | — | +| all-readonly-cleanstack | 777 | 539 | 252 (32.4%) | 288 (37.1%) | 14 (1.8%) | 146 (18.8%) | 4 (0.5%) | 72 (9.3%) | 1 (0.1%) | — | +| babybear | 647 | 351 | 370 (57.2%) | 99 (15.3%) | 156 (24.1%) | — | — | 7 (1.1%) | 15 (2.3%) | — | +| if-without-else-multi-temp | 226 | 219 | 11 (4.9%) | 125 (55.3%) | 15 (6.6%) | 24 (10.6%) | — | 25 (11.1%) | 26 (11.5%) | — | +| merkle-proof | 201 | 193 | 16 (8.0%) | 108 (53.7%) | 16 (8.0%) | 18 (9.0%) | 8 (4.0%) | 16 (8.0%) | 19 (9.5%) | — | +| bitwise-ops | 96 | 96 | — | 34 (35.4%) | 26 (27.1%) | — | — | 27 (28.1%) | 9 (9.4%) | — | +| cross-covenant | 46 | 45 | 2 (4.3%) | 24 (52.2%) | 3 (6.5%) | 8 (17.4%) | 2 (4.3%) | 4 (8.7%) | 3 (6.5%) | — | +| oracle-price | 44 | 38 | 8 (18.2%) | 21 (47.7%) | 5 (11.4%) | 2 (4.5%) | 2 (4.5%) | 4 (9.1%) | 2 (4.5%) | — | +| bounded-loop | 42 | 42 | — | 24 (57.1%) | 11 (26.2%) | — | — | 7 (16.7%) | — | — | +| arithmetic | 28 | 28 | — | 19 (67.9%) | 8 (28.6%) | — | — | 1 (3.6%) | — | — | +| if-without-else | 27 | 27 | — | 10 (37.0%) | 5 (18.5%) | — | — | 6 (22.2%) | 6 (22.2%) | — | +| shift-ops | 27 | 27 | — | 10 (37.0%) | 8 (29.6%) | — | — | 8 (29.6%) | 1 (3.7%) | — | +| if-else | 20 | 20 | — | 10 (50.0%) | 3 (15.0%) | — | — | 4 (20.0%) | 3 (15.0%) | — | +| escrow | 19 | 19 | — | 4 (21.1%) | 2 (10.5%) | — | 4 (21.1%) | 6 (31.6%) | 3 (15.8%) | — | +| multi-method | 19 | 19 | — | 2 (10.5%) | 5 (26.3%) | — | 2 (10.5%) | 6 (31.6%) | 4 (21.1%) | — | +| multisig | 17 | 17 | — | 10 (58.8%) | — | — | 1 (5.9%) | 6 (35.3%) | — | — | +| boolean-logic | 15 | 15 | — | 6 (40.0%) | 7 (46.7%) | — | — | 2 (13.3%) | — | — | +| go-dsl-bytestring-literal | 8 | 6 | 3 (37.5%) | — | 2 (25.0%) | 1 (12.5%) | — | 2 (25.0%) | — | — | +| basic-p2pkh | 5 | 5 | — | 1 (20.0%) | — | 1 (20.0%) | 2 (40.0%) | 1 (20.0%) | — | — | +| asm-raw-script | 1 | 1 | — | — | — | — | — | 1 (100.0%) | — | — | +_`asm-raw-script` is a single opaque `raw_bytes` span; `basic-p2pkh` is the 5-byte template +before constructor-arg splicing. Neither is a size target._ + +--- + +## 7. Method + +`analyzeScriptHex` walks the serialized script and charges every byte to exactly one +category; the sum is asserted equal to the script length. One rule is worth stating: a push +immediately consumed by `OP_PICK` / `OP_ROLL` is charged to **stack-shuffle**, not to +**const-push**. `bringToTop` emits `push(depth)` + `OP_PICK` as a pair +(`05-stack-lower.ts:1062`), and charging those depth bytes to constants would credit the +wrong optimizer with removing them. On `p256-wallet` that reclassification moves exactly +21,926 bytes, and it carries a second useful fact: all 21,926 of those depth pushes are a +single byte, so the EC macros never `OP_PICK` deeper than 16. A pooled constant parked below +a working set that shallow would cost 3 bytes to copy (2-byte depth push + `OP_PICK`) instead +of 34 — still a 31-byte saving per reduction. + +Categories: `const-push` (length-prefixed / PUSHDATA payloads), `small-int-push` +(OP_0/OP_1NEGATE/OP_1..16), `stack-shuffle` (DUP/DROP/NIP/OVER/PICK/ROLL/ROT/SWAP/TUCK/2DROP/ +2DUP/3DUP/2OVER/2ROT/2SWAP/IFDUP/DEPTH/TOALTSTACK/FROMALTSTACK plus PICK/ROLL depth pushes), +`arithmetic` (numeric, bitwise and comparison opcodes), `bytes` (CAT/SPLIT/SIZE/NUM2BIN/ +BIN2NUM/SUBSTR/LEFT/RIGHT/EQUAL/EQUALVERIFY), `crypto` (hashes, CHECKSIG family, +CODESEPARATOR), `control` (IF/NOTIF/ELSE/ENDIF/VERIFY/RETURN/NOP/CLTV/CSV), `other`. diff --git a/docs/experiments/script-size-optimizer-results.md b/docs/experiments/script-size-optimizer-results.md new file mode 100644 index 00000000..eb538164 --- /dev/null +++ b/docs/experiments/script-size-optimizer-results.md @@ -0,0 +1,288 @@ +# Script-size optimizer — Phases 0–2 results + +**Scope:** the first slice of the size-optimization brief — baseline instrumentation (Phase 0), +an exact script-byte cost model (Phase 1), and two prototype optimizations behind opt-in flags +(Phase 2). No modular-domain analysis, no witness hints, no scalar-multiplication algorithms. +**Companion documents:** [`script-size-optimization-baseline.md`](script-size-optimization-baseline.md), +[`stack-scheduler-design.md`](stack-scheduler-design.md). + +Everything below is measured on the 72 conformance fixtures with +`pnpm --filter runar-conformance run script-metrics -- --compare current,liveness,ec-pool,both`. + +--- + +## 1. Headline + +| | corpus bytes | vs baseline | fixtures changed | fixtures grown | +|---|---:|---:|---:|---:| +| `current` (shipping) | 13,526,563 | — | — | — | +| `liveness` (scheduler) | 13,526,482 | −0.0 % | 34 | 0 | +| `ec-pool` (constant pool) | 6,285,154 | **−53.5 %** | 9 | 0 | +| `both` | 6,285,073 | **−53.5 %** | 43 | 0 | + +**`conformance/tests/p256-wallet`: 958,792 → 304,463 bytes (−68.2 %)**, which is the brief's +959 kB reference implementation. `p384-wallet`: 1,963,300 → 463,435 (−76.4 %). + +Default output is unchanged: all 72 fixtures still reproduce their checked-in +`expected-script.hex` byte-for-byte +(`packages/runar-compiler/src/__tests__/golden-invariance.test.ts`), and the Go and Rust +cross-compiler golden tests still pass. + +--- + +## 2. What was built + +| Phase | Deliverable | Files | +|---|---|---| +| 0 | Byte-category instrumentation + benchmark runner | `packages/runar-compiler/src/metrics/script-metrics.ts`, `conformance/runner/script-metrics.ts` | +| 1 | Exact script-byte cost model | `packages/runar-compiler/src/metrics/cost-model.ts` | +| 2a | Liveness scheduler (`--stack-scheduler=liveness`) | `packages/runar-compiler/src/passes/05-stack-lower.ts` | +| 2b | EC constant pool (`--ec-constant-pool`) | `packages/runar-compiler/src/passes/ec-codegen.ts`, `p256-p384-codegen.ts` | + +### Phase 1 — the cost model is exact, not an estimate + +```ts +estimateScriptBytes(ops) === emitMethod({ ops, … }).scriptHex.length / 2 +``` + +is asserted for every method of every fixture that ships a `.runar.ts`, before and after +peephole (`__tests__/cost-model.test.ts`, 101 assertions). That exactness turned out to matter +more than expected — see §5. + +### Phase 2b — EC constant pool + +`ECTracker` gained a pooled slot per curve constant. `pushConst(slot, value, name)` compares +the emitted cost of `OP_PICK`-ing the slot against re-pushing the literal and takes the +cheaper, so pooling can never make an individual call site larger. Every EC emitter that does +more than a couple of reductions parks the field prime (and, for the ladders, the group order) +on entry and releases it on exit. + +| emitter | current | pooled | | +|---|---:|---:|---:| +| `emitVerifyECDSA_P256` | 974,024 | 319,693 | −67.2 % | +| `emitVerifyECDSA_P384` | 1,987,394 | 487,527 | −75.5 % | +| `emitP256Mul` | 459,746 | 150,512 | −67.3 % | +| `emitP384Mul` | 927,350 | 227,044 | −75.5 % | +| `emitEcMul` (secp256k1) | 428,676 | 140,242 | −67.3 % | +| `emitEcAdd` | 25,426 | 8,791 | −65.4 % | + +### Phase 2a — liveness scheduler + +Two transformations, both gated on `schedulerMode: 'liveness'`: + +1. **Alt-stack result spilling.** A result whose next use is not the following binding is + parked with `OP_TOALTSTACK`, keeping the operands a chain reads repeatedly at depth 0/1. + The whole spill group is restored in one go before the first binding that needs any of it — + which puts the values back in production order, the order an ANF accumulation reads them. +2. **Commutative operand ordering.** For `+ * === !== & | ^` (excluding `+` on ByteString, + which is `OP_CAT`), the operands are materialized in whichever order the cost model scores + cheaper. + +Selection is per method: both schedules are lowered and the cheaper one — measured after +peephole — is kept. "The scheduler never grows a method" is therefore structural, not a hope. + +| fixture | current | liveness | | +|---|---:|---:|---:| +| `arithmetic` | 28 | 18 | **−35.7 %** | +| `bounded-loop` | 42 | 37 | −11.9 % | +| `if-without-else-multi-temp` | 244 | 243 | −0.4 % | +| ~30 mid-size fixtures | 795–17,348 | | −0.1 % to −0.2 % | +| EC / SLH-DSA / SHA-256 / BLAKE3 | | unchanged | 0 % | + +18 bytes for `arithmetic` is the hand-derived optimum recorded in +[`stack-scheduler-design.md`](stack-scheduler-design.md) §2.4 — the scheduler reached it +independently. + +--- + +## 3. Answers to the brief's questions (for this slice) + +### 3.1 What is generic? + +- **The byte-cost model.** Nothing else in the compiler could compare two lowerings by the + metric that matters. It should become permanent infrastructure regardless of which + optimizations ship. +- **Cost-model-driven selection.** Lowering a method both ways and keeping the smaller is + cheap, obviously correct, and removes a whole class of "the heuristic guessed wrong" + regressions. Recommended as the general pattern for any future scheduling change. +- **Commutative operand ordering.** Small, local, and applies to every contract. +- **Constant pooling as a *policy*** — "if a value is materialized more than + `pool_cost / (push_cost − pick_cost)` times, park it" — is fully generic even though this + implementation applies it inside the EC macros. + +### 3.2 What needs explicit programmer intent? + +Nothing in this slice. Both flags are compiler-internal and semantics-preserving. Witness +hints (brief Phase 7) remain the first thing that genuinely requires a contract-level opt-in. + +### 3.3 What belongs in crypto libraries / intrinsics? + +The pooling *mechanism* had to live in `ECTracker`, because the crypto emitters build their +own stack layout and never pass through `05-stack-lower.ts`. That boundary is the single most +important architectural fact this slice established — see §4. + +### 3.4 What is P-256-specific? + +Nothing that was built. `POOL_FIELD_P` / `POOL_GROUP_N` are parameterized by `CurveParams` / +`GroupParams`; the same code path serves secp256k1, P-256 and P-384, and the −67 % / −76 % +results differ only because P-384's prime is a 50-byte push instead of 34. + +### 3.5 What should not be implemented? + +- **Eager dead-slot retirement.** The plan assumed dead values sink and inflate later + `OP_PICK` depth pushes. Measurement killed it: of 387,749 `OP_PICK`/`OP_ROLL` sites in the + corpus, **657 are deeper than 16**; typical depths are 2–5, and depths 0–2 are single-byte + opcodes anyway. Every drop would cost 1–3 bytes to save approximately nothing. +- **A forked scheduler pass.** A `schedulerMode` field on `LoweringContext` kept one code + path and let the existing 66 structural assertions in `05-stack-lower.test.ts` cover both + modes. A 5,500-line fork would have drifted from the branch/loop invariant fixes landing on + the original. + +### 3.6 Can Rúnar approach the ~29.6 kB P-256 result? + +Not yet — `p256-wallet` is at 304,463 bytes, 10× the target. But the remaining bytes are now +in exactly two buckets, and neither is mysterious: + +| category | bytes | share | +|---|---:|---:| +| stack-shuffle | 214,904 | 70.6 % | +| arithmetic | 82,863 | 27.2 % | +| small-int push | 2,698 | 0.9 % | +| const-push | 1,753 | 0.6 % | +| everything else | 2,245 | 0.7 % | + +Constant pushes went from 697,019 bytes to 1,753 — 99.7 % eliminated. What is left is the +modular-reduction sequence itself, repeated ~20,000 times: + +``` +pick p 2 bytes (was a 34-byte push) +OP_2DUP OP_MOD OP_ROT OP_DROP OP_OVER OP_ADD OP_SWAP OP_MOD 8 bytes +``` + +The six-opcode tail after the first `OP_MOD` exists only because `OP_MOD` takes the sign of +the dividend. For a product of two values already reduced into `[0, p)` the dividend is +non-negative and the tail is dead weight. Knowing that requires modular-domain analysis — +**brief Phase 4/5, and now clearly the single highest-value next step**: + +``` +304,463 today + − ~140,000 drop the sign-normalisation tail where the domain proves it dead +≈ 164,000 after lazy/typed modular reduction + − ~80,000 Straus/Shamir: one joint ladder instead of two (brief Phase 9) +≈ 84,000 + − ~50,000 fixed-base comb for u1·G, G being compile-time known (Phases 10–11) +≈ 34,000 ← the reference trajectory's comb stage (34,470 B) +``` + +That projection lands on the reference numbers, which is a reason to believe the remaining +gap is algorithmic rather than structural. It is a projection, not a measurement. + +--- + +## 4. The architectural finding + +**Crypto codegen does not pass through the generic backend.** `ec-codegen.ts`, +`p256-p384-codegen.ts`, `sha256-codegen.ts`, `slh-dsa-codegen.ts`, `babybear-codegen.ts` and +their peers emit Stack IR directly through hand-written `ECTracker`-family trackers. +`05-stack-lower.ts` never sees those ops. + +Consequences, all confirmed by measurement: + +- A generic scheduler cannot move a single byte of the EC, SLH-DSA, SHA-256 or BLAKE3 + fixtures — 13.4 MB of the 13.5 MB corpus. +- Conversely, the constant pool cannot help ordinary contracts. +- Any future "generic" optimization needs an answer to which of the two worlds it lives in + before its value can be estimated. + +The peephole optimizer is the one pass that spans both, because it runs on the whole method's +Stack IR after lowering. That makes new peephole rules unusually high-leverage — and it is +already a fix-point driver (brief Phase 3 is done; see the baseline document §5). + +--- + +## 5. What the oracles caught + +The scheduler's first working version **miscompiled `if-without-else-multi-temp`**: it +produced a script that ran to completion, left a truthy top-of-stack, and **accepted a witness +the shipping compiler rejects**. Byte counts, golden comparisons for the other fixtures, and +the compiler's own 4,099 unit tests all passed while this was true. + +What caught it was `conformance/witnesses/` replayed through `runDifferentialExecution` — +source-semantics interpreter versus deployed script, on witnesses the repo had already +committed to, with at least one accept and one reject per fixture. Two of 86 cases failed. + +Root cause: restoring spilled values immediately before an `if` leaves the parent stack in a +shape `lowerIf`'s arm reconciliation was not written for. The fix is a precondition — spilling +is refused in any scope that still has control flow ahead of it — not an attempt to make the +two agree. That costs the scheduler its wins on `oracle-price` and `cross-covenant` (2 and 1 +bytes) and keeps everything else. + +Two process notes worth carrying forward: + +- A bisect that "passes" can be vacuous. Disabling commutative reordering made the failure + disappear, which looked like an acquittal for spilling — but with reordering off, the + method-level cost guard simply preferred the baseline schedule and *no spilling happened at + all*. Confirming that the variant actually changed the bytes was what made the second + bisect meaningful. +- The per-site cost model was wrong until it was made peephole-aware. Two consumed operands at + depths 1 and 0 emit `OP_SWAP OP_SWAP`, which the `swap-swap` rule deletes outright — free — + while the "cheaper-looking" order emits one real `OP_SWAP`. Scoring candidate op sequences + through `optimizeStackIR` before comparing them took `arithmetic` from 24 bytes to 18. + +--- + +## 6. Recommended compiler changes + +**Adopt now (independent of any optimization):** + +1. `packages/runar-compiler/src/metrics/cost-model.ts` and `script-metrics.ts` as permanent + infrastructure, with the exactness sweep as a standing test. +2. `conformance/runner/script-metrics.ts` alongside the existing `script-size-check.ts` — the + latter answers "did anything grow?", the former "where did the bytes go?". + +**Adopt after a 7-tier port:** + +3. The EC constant pool. It is the largest single byte win available, it is + curve-parameterized rather than curve-specific, and it is proved equivalent against OpenSSL + signatures on both curves plus every SEC1 rejection case + (`packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts`, 44 cases). + Landing it means porting to `compilers/{go,rust,python,ruby,zig,java}`, regenerating 9 + goldens, re-stamping `conformance/script-size-baseline.json` (the −67 % shrink trips its + 50 % guard by design), and adding provenance entries. + +**Keep experimental:** + +4. The liveness scheduler. It is correct and never grows a fixture, but outside small + arithmetic contracts it buys 0.1 %. Not worth a 7-tier port on its own; worth keeping as a + gated mode so the next scheduling idea has somewhere to land. + +**Do next (highest value first):** + +5. **Modular-domain analysis + reduction sinking** (brief Phases 4–5). ~140 kB on + `p256-wallet`, and it is the prerequisite for everything after it. +6. **Straus/Shamir joint ladder**, then a **fixed-base comb** for `u1·G` (Phases 9–11). +7. **Witness-hint modular inverse** (Phase 7) — removes three unrolled Fermat ladders + (382 + 423 + 286 field multiplications per P-256 verify). + +--- + +## 7. Reproducing + +```bash +# Sizes +pnpm --filter runar-conformance run script-metrics # baseline table +pnpm --filter runar-conformance run script-metrics -- --fixture p256-wallet --detail +pnpm --filter runar-conformance run script-metrics -- --compare current,liveness,ec-pool,both + +# Correctness +npx vitest run packages/runar-compiler/src/__tests__/cost-model.test.ts # model is exact +npx vitest run packages/runar-compiler/src/__tests__/golden-invariance.test.ts # default unchanged +npx vitest run packages/runar-compiler/src/__tests__/ec-constant-pool.test.ts +npx vitest run packages/runar-compiler/src/__tests__/liveness-scheduler.test.ts +npx vitest run packages/runar-testing/src/__tests__/ec-constant-pool-equivalence.test.ts +npx vitest run packages/runar-testing/src/__tests__/liveness-scheduler-equivalence.test.ts +npx vitest run packages/runar-testing/src/__tests__/scheduler-headroom.test.ts + +# CLI +node --import tsx packages/runar-cli/src/bin.ts compile --ec-constant-pool --hex +``` diff --git a/docs/experiments/stack-scheduler-design.md b/docs/experiments/stack-scheduler-design.md new file mode 100644 index 00000000..4905e4b9 --- /dev/null +++ b/docs/experiments/stack-scheduler-design.md @@ -0,0 +1,393 @@ +# Stack scheduler — current behaviour, measured inefficiencies, and a design + +**Status:** design, pre-implementation. Companion to +[`script-size-optimization-baseline.md`](script-size-optimization-baseline.md). +**Scope:** the generic ANF → Stack lowering in +`packages/runar-compiler/src/passes/05-stack-lower.ts`. Crypto macro modules +(`ec-codegen.ts`, `p256-p384-codegen.ts`, `sha256-codegen.ts`, `slh-dsa-codegen.ts`, …) +emit Stack IR directly through their own `ECTracker`-family trackers and are **out of +scope for this document** — they are addressed separately by constant pooling. + +--- + +## 1. What the current lowering does + +### 1.1 The symbolic stack + +`class StackMap` (`05-stack-lower.ts:166`) is an array of `(string | null)`, index 0 = +bottom. `null` marks an anonymous slot (e.g. the discard half of an `OP_SPLIT`). + +```ts +findDepth(name): number // :192 — searches TOP-DOWN, returns depth-from-top +removeAtDepth(d) // :207 — the ROLL effect +peekAtDepth(d) // :217 — the PICK effect +clone() // :226 — used to fork a branch arm +``` + +`findDepth` resolves to the *shallowest* match, so rebinding a name pushes a new slot with +the same name and the old one becomes dead-but-resident. `LoweringContext` (`:547`) owns one +`StackMap`, the emitted `ops`, `maxDepth`, and `outerProtectedRefs`. The constructor (`:578`) +seeds the map with parameter names, first param at the bottom. + +### 1.2 Liveness + +The entire liveness analysis is `computeLastUses(bindings)` (`:276`): one forward scan +mapping each referenced name → the highest binding index that references it. Array-literal +indirection is patched through (`:284`) so element temps stay live to the array's consumer. + +Consumption is decided by two predicates: + +```ts +isLastUse(ref, i, lastUses) // :1304 last <= i +operandConsume(ref, operands, i, …) // :1328 isLastUse AND appears once in this operand list +``` + +`operandConsume` needs the occurrence check because `t := x + x` must PICK at both positions. + +Outer-scope values are pinned by *forcing* their last use past the end: +`lastUses.set(ref, bindings.length)` (`:1154`, and `:2196` for branch arms). That is the only +pinning mechanism. + +### 1.3 Materialization — `bringToTop(name, consume)` (`:1038`) + +Every operand goes through this one function: + +| depth | consume (last use) | !consume (still live) | +|---:|---|---| +| 0 | nothing | `OP_DUP` | +| 1 | `OP_SWAP` | `OP_OVER` | +| 2 | `OP_ROT` | `push 2; OP_PICK` | +| d | `push d; OP_ROLL` | `push d; OP_PICK` | + +The depth 0/1/2 peepholes are inlined here, which is why the peephole rules `roll1-to-swap`, +`roll2-to-rot`, `pick0-to-dup`, `pick1-to-over` almost never fire on the main path. + +### 1.4 What is *not* done + +- **No operand reordering.** `lowerBinOp` (`:1507`) always materializes left then right, + even for commutative operators, and `lowerCall` (`:1839`) always walks args in order. +- **No proactive dead-value removal.** `computeLastUses` knows exactly when a temp dies; + nothing acts on it. Dead slots linger until `cleanupExcessStack()` (`:621`) NIPs the method + tail, or until a branch's `drainBranchPrivateResidue` (`:1112`) sweeps them. +- **No alt stack.** The generic lowerer emits `OP_TOALTSTACK` in exactly three places + (`:3013`, `:3105` state serialization, and the `divmod` intrinsic at `:4517`). Never for + scheduling. `docs/compiler-architecture.md` claims otherwise — that paragraph is + aspirational and should be corrected. +- **No cost model.** Choices are structural, never compared by emitted bytes. + +### 1.5 Branches and loops, in one line each + +`lowerIf` (`:2092`) forks a cloned `StackMap` per arm, pins every parent value that outlives +the `if`, reconciles asymmetric consumption, trims to the declared `results` layout, pads the +shallower arm with 1-byte empty pushes (`:2400`, `:2405`), and asserts equal arm depth at +`OP_ENDIF`. `lowerLoop` (`:2665`) fully unrolls, recomputing `lastUses` per iteration and +pinning loop-carried refs on every non-final iteration. + +Any scheduler change must leave these invariants intact — they are enforced by hard throws +(`branch result layout mismatch` at `:2346`, the Layer B/C depth assertions at `:2417`, +`:2640`), not by tests alone. + +--- + +## 2. Measured inefficiencies + +All figures from the 72 checked-in goldens via +`pnpm --filter runar-conformance run script-metrics`. + +### 2.1 Stack traffic is 23 % of the corpus — and 35–68 % of ordinary contracts + +| fixture | bytes | stack-shuffle share | +|---|---:|---:| +| `arithmetic` | 28 | **67.9 %** | +| `bounded-loop` | 42 | 57.1 % | +| `multisig` | 17 | 58.8 % | +| `if-without-else-multi-temp` | 226 | 55.3 % | +| `stateful-counter` | 1,875 | 35.9 % | +| `token-ft` | 3,154 | 36.1 % | +| `math-demo` | 17,348 | 35.7 % | + +### 2.2 The dead-slot hypothesis is **refuted** + +The obvious theory — dead values sink under live ones, so later accesses pay a deeper +`push(depth)`, and crossing depth 16 turns a 1-byte depth push into 2 — does not survive +measurement. Across all 387,749 `OP_PICK`/`OP_ROLL` sites in the corpus: + +``` +depth ≤ 16 : 387,092 (1-byte depth push) +depth > 16 : 657 (2-byte depth push) +deepest anywhere: 75 +``` + +Typical depths are 2–5. On `p256-wallet` every one of the 21,926 depth pushes is a single +byte. **Eager dead-slot retirement would cost 1–3 bytes per drop to save essentially nothing, +and is dropped from the design.** This is the main correction to the original plan. + +### 2.3 Where the shuffle bytes actually are + +`p256-wallet`: 173,967 shuffle bytes, of which only 43,852 are `PICK`/`ROLL` (op + depth +push). The remaining ~130 kB is bare one-byte shuffles — `OP_ROT`×30,406, `OP_SWAP`×27,342, +`OP_OVER`×23,417, `OP_DROP`×22,558, `OP_2DUP`×20,453 — and ~100 kB of that is the fixed +five-shuffle tail inside `cFieldMod`, i.e. crypto-macro output, not this scheduler. + +For the ordinary contracts the picture inverts: `arithmetic` spends 16 of 28 bytes on stack +access, split 8 bytes of depth pushes and 8 bytes of `PICK`/`ROLL`/`ROT`/`SWAP`. + +### 2.4 A measured headroom number + +`conformance/tests/arithmetic` is the only fixture whose bytes are produced *entirely* by +this pass. Source: + +```ts +const sum = a + b; const diff = a - b; const prod = a * b; const quot = a / b; +assert(sum + diff + prod + quot === this.target); +``` + +Emitted today (28 bytes, `00` = constructor placeholder): + +``` +OP_2DUP OP_ADD sum [a,b,sum] +OP_2 OP_PICK OP_2 OP_PICK OP_SUB diff [a,b,sum,diff] +OP_3 OP_PICK OP_3 OP_PICK OP_MUL prod [a,b,sum,diff,prod] +OP_4 OP_ROLL OP_4 OP_ROLL OP_DIV quot [sum,diff,prod,quot] +OP_3 OP_ROLL OP_3 OP_ROLL OP_ADD OP_ROT OP_ADD OP_SWAP OP_ADD + OP_NUMEQUAL +``` + +`a` and `b` are each materialized four times, and every materialization is deeper than the +last because each result is pushed on top of them. + +Hand-scheduled alternative (18 bytes) — operands stay in the top two slots, finished results +are parked on the alt stack: + +``` +OP_2DUP OP_ADD OP_TOALTSTACK sum -> alt +OP_2DUP OP_SUB OP_TOALTSTACK diff -> alt +OP_2DUP OP_MUL OP_TOALTSTACK prod -> alt +OP_DIV quot (consumes a, b) +OP_FROMALTSTACK OP_ADD + prod +OP_FROMALTSTACK OP_ADD + diff +OP_FROMALTSTACK OP_ADD + sum + OP_NUMEQUAL +``` + +**28 → 18 bytes, −36 %, and every byte saved is stack traffic.** Both scripts are executed +against the real `@bsv/sdk` interpreter over 14 (a, b) pairs including negatives, zero, and +the 16/17 script-number boundary, in +`packages/runar-testing/src/__tests__/scheduler-headroom.test.ts`; they accept and reject +identically. The number is measured, not estimated. + +The three effects that produced it, in order of contribution: + +1. **Result spilling to the alt stack.** Keeps the hot operands at depth 0/1, so every + subsequent access is `OP_2DUP` (1 byte) instead of `push d; OP_PICK; push d; OP_PICK` + (4 bytes). Worth 3 of the 4 materializations here. +2. **Adjacent-pair fusion.** `(a, b)` at depths 1 and 0 is `OP_2DUP`, not two picks. The + peephole has an `over,over → OP_2DUP` rule, but the lowerer emits `push;pick;push;pick`, + which no window can fuse. +3. **Consumption ordering.** Scheduling the one operator that *consumes* `a` and `b` + (`OP_DIV`) last removes the final pair of `OP_4 OP_ROLL`s entirely. + +--- + +## 3. Design + +### 3.1 Representation + +Extend the existing analysis rather than replacing it. `computeLastUses` already gives last +use; add, over the same scan: + +```ts +interface LivenessInfo { + lastUse: Map; // existing + useCount: Map; // total references (excluding array indirection) + nextUse: Map; // sorted binding indices that reference the name +} +``` + +`nextUse` is what makes "will this value be touched again soon, or not for a while?" a +question the scheduler can answer, and it is the input to the spill decision. It costs one +extra pass over the same bindings. + +### 3.2 Scheduling unit: the branch-free run + +The prototype operates only on a **maximal run of consecutive bindings containing no `if` +and no `loop`**. A run ends at any control-flow binding, and the stack is restored to a +canonical layout (everything on the main stack, alt stack empty) before that binding is +lowered. + +This is deliberate. `lowerIf`'s arm reconciliation, result-layout assertion and depth +balancing (§1.5) are the most delicate code in the backend, and every one of them reasons +about main-stack depth only. Confining spills to branch-free runs means **no arm ever begins +or ends with a non-empty alt stack**, so none of those invariants can be perturbed. It also +means the prototype cannot help inside a loop body — accepted for now; loops are unrolled, so +the runs *between* control flow are still scheduled. + +### 3.3 Byte-cost function + +`estimateScriptBytes` / `sizeOfStackOp` from `packages/runar-compiler/src/metrics/cost-model.ts` +— already implemented and asserted byte-exact against `06-emit.ts` over the whole corpus +(`__tests__/cost-model.test.ts`). The scheduler's local decisions use these derived costs: + +``` +accessCost(depth, consume) = 1 depth 0 (consume) — free + = 1 depth 0/1/2 via DUP/SWAP/OVER/ROT + = sizeOfPushValue(depth)+1 otherwise +pairAccessCost(d0, d1) = 1 (a,b) at depths 1,0 -> OP_2DUP + = accessCost(d0)+accessCost(d1) otherwise +spillCost = 2 TOALTSTACK + FROMALTSTACK +rematerializeCost(constant) = sizeOfPushValue(v) +``` + +### 3.4 Heuristic + +Greedy, single forward pass over a run. Not globally optimal, and deliberately so — the +brief asks for a simple greedy implementation first. + +For each binding `t := op(x, y)`: + +1. **Order the operands.** If `op` is commutative (`+ * === !== && || & | ^`, `min`, `max`), + order so the operand already nearer the top is materialized second. Ties keep source + order, so the default mode is unchanged. +2. **Fuse the pair.** If `(x, y)` sit at depths 1 and 0 and neither is consumed, emit + `OP_2DUP` instead of two accesses. Generalize to `OP_2OVER` for depths 3,2. +3. **Rematerialize instead of accessing.** If `x` is a `load_const` whose push encoding costs + ≤ `accessCost(depth(x), consume)`, re-push it and leave the resident copy alone. +4. **Spill the result.** After emitting the operation, if the result's `nextUse` is more than + `SPILL_HORIZON` bindings away *and* at least one still-live value sits below it, park it + with `OP_TOALTSTACK`. Restore in reverse spill order at the point of use. Spill only when + `spillCost < projected access savings`, computed from `nextUse` and the current depths. +5. **Restore before a run boundary.** Every spilled value is popped back before any `if`, + `loop`, or the end of the method. + +**As built, step 4 is stricter than this design anticipated.** Spilling is refused outright in +any scope that still has control flow ahead of it, because restoring immediately before an +`if` miscompiled a fixture — see §6. And step 1 is scored by running the candidate op +sequences through the real peephole rather than a byte formula, because the cheapest-looking +local choice is often one the peephole would have erased anyway (§6 again). + +The per-site heuristics are backed by a **method-level guard**: both schedules are lowered and +the cheaper one, measured after peephole with `estimateScriptBytes`, is kept. "The scheduler +never grows a method" is therefore a structural property, not a hope — which matters, because +the greedy heuristic cannot tell whether removing one slot actually moves an access across a +cost boundary (depths 0-2 are all one byte). + +### 3.5 Gating + +`schedulerMode: 'current' | 'liveness'` on `LoweringContext`, plumbed from +`CompileOptions.schedulerMode` (`packages/runar-compiler/src/index.ts:109`) and a CLI +`--stack-scheduler=` (`packages/runar-cli/src/bin.ts:39`, +`commands/compile.ts:14/84/177/252` — the `--disable-constant-folding` path is the template). + +Default is `'current'`, and every new behaviour is a no-op in that mode. This keeps the +72 goldens, `conformance/script-size-baseline.json`, the cross-tier hex parity gate and the +golden-provenance gate untouched while the experiment runs. + +--- + +## 4. Correctness invariants + +The scheduler may reorder *materialization*, never *evaluation*. Concretely: + +1. **Side-effect order is fixed.** Bindings are lowered in ANF order. Only the stack + operations that arrange operands may move. `hasSideEffect` (`optimizer/dce.ts:133`) names + the kinds that must never be reordered relative to each other. +2. **Operand order is preserved for non-commutative operators.** `-`, `/`, `%`, `<<`, `>>`, + `<`, `>`, `<=`, `>=`, `OP_SPLIT`, `OP_CAT` and every intrinsic keep source order. + Commutativity is asserted per-operator against the interpreter, not assumed: `OP_ADD` and + `OP_MUL` are commutative on script numbers; `OP_CAT` is not; `OP_BOOLAND`/`OP_BOOLOR` are + commutative but **not** short-circuit at this level, so reordering them cannot change + which side is evaluated (both already are). +3. **Alt stack is empty at every control-flow boundary** and at method exit. Asserted in the + lowerer, not just tested — a `LoweringContext` invariant check before each `if`/`loop` and + in `lowerMethod`. +4. **Main-stack depth at `OP_ENDIF` is unchanged.** The Layer B/C assertions (`:2417`, + `:2640`) stay in force; the prototype must not touch arm reconciliation at all. +5. **`maxStackDepth` may not exceed `MAX_STACK_DEPTH = 800`** (`:63`). Spilling *reduces* + main-stack depth, but the alt stack shares the interpreter's 1,000-element budget, so the + sum is what gets checked. +6. **No assertion is weakened.** The scheduler never removes an `assert` binding, never + changes which value an `OP_VERIFY` consumes, and never elides a normalization + (`OP_BIN2NUM`, `OP_NUM2BIN`, sign fixups) that a later consumer observes. + +--- + +## 5. Benchmark plan + +**Metric:** serialized locking-script bytes, from `estimateScriptBytes` (exact) and confirmed +against the emitted hex. + +**Command:** + +```bash +pnpm --filter runar-conformance run script-metrics -- --compare current,liveness +``` + +**Report, per fixture:** script bytes, `OP_PICK` / `OP_ROLL` / `OP_DUP` / `OP_SWAP` / +`OP_2DUP` / `OP_TOALTSTACK` counts, `maxStackDepth` delta. + +**Acceptance (from the brief, restated as pass/fail):** + +1. Semantically identical Script — proven, not assumed. `scheduler-equivalence.test.ts` + (modelled on `packages/runar-testing/src/oracle/fold-equivalence.ts`) compiles each source + under both modes and asserts identical accept/reject through `ScriptVM` plus agreement + with the mode-independent AST interpreter, over every witness in + `conformance/witnesses/`. Plus `conformance/fuzzer/index.ts --execute` with the toggle. +2. All existing VM / interpreter tests pass under both modes. +3. No material growth on ordinary fixtures. Fail the experiment if any fixture grows > 1 %. +4. A measurable win on at least one arithmetic-heavy fixture. **Target: > 10 %.** + +**Prior expectations, so the result could disappoint honestly:** + +| fixture class | expected | **measured** | +|---|---|---| +| `arithmetic` | −20 % to −36 % | **−35.7 %** (28 → 18 B) | +| `bounded-loop`, `boolean-logic` | −20 % to −36 % | −11.9 % (42 → 37 B) / 0 % | +| `math-demo`, `token-ft`, `function-patterns` | −3 % to −10 % | **−0.1 %** | +| stateful fixtures | −1 % to −4 % | −0.1 % to −0.2 % | +| EC / P-256 / P-384 | 0 % | 0 % | +| SLH-DSA / SHA-256 / BLAKE3 | 0 % | 0 % | + +`arithmetic` reached 18 bytes — the hand-derived optimum in §2.4 — which the scheduler found +on its own. The mid-size prediction was wrong by an order of magnitude: those contracts do have +a 35 % stack-shuffle share, but almost all of it is sighash and state-serialization macro +output, not ANF chains the scheduler can reach. Their ANF is mostly bindings consumed by the +very next binding, where there is nothing to spill. + +So the honest conclusion is the one the plan named as the disappointing case: **the generic +scheduler is worth having for small arithmetic contracts and little else**, and the remaining +shuffle budget belongs to the macro emitters. Corpus-wide it moves 34 of 72 fixtures and +−0.0 % of total bytes. It is kept as a gated mode, not proposed for a 7-tier port. + +--- + +## 6. What went wrong, and what caught it + +### A miscompile, caught by the witness corpus + +The first working scheduler **miscompiled `if-without-else-multi-temp`**: the script ran to +completion, left a truthy top-of-stack, and **accepted a witness the shipping compiler +rejects**. Byte counts, the goldens for every other fixture, and 4,099 compiler unit tests all +passed while that was true. + +`conformance/witnesses/` replayed through `runDifferentialExecution` caught it — deployed +script versus the ANF interpreter, on witnesses the repo had already committed to, with at +least one accept and one reject per fixture. Two of 86 cases failed. + +Cause: restoring spilled values immediately before an `if` leaves the parent stack in a shape +`lowerIf`'s arm reconciliation, declared-result trim and Layer B/C depth invariants were not +written for. The fix is the precondition in §3.4 — refuse to spill in a scope with control +flow ahead of it — rather than an attempt to make the two agree. + +### Two things worth remembering + +**A passing bisect can be vacuous.** Turning off commutative reordering made the failure +disappear, which looked like an acquittal for spilling. It was not: with reordering off, the +method-level cost guard simply preferred the baseline schedule, so no spilling happened at +all. Only after confirming the variant still changed the emitted bytes did the second bisect +mean anything. + +**The cost model had to be peephole-aware.** Two consumed operands at depths 1 and 0 emit +`OP_SWAP OP_SWAP`, which the `swap-swap` rule deletes outright — free — while the +"cheaper-looking" reversed order emits one real `OP_SWAP` and costs a byte. Scoring candidate +op sequences through `optimizeStackIR` before comparing them took `arithmetic` from 24 bytes +to 18. From b345272b5ecede4a762e92d42d92e20690dddf40 Mon Sep 17 00:00:00 2001 From: Siggi Date: Fri, 28 Aug 2026 23:22:09 +0200 Subject: [PATCH 4/4] docs(experiments): measure the reduction-sinking ceiling instead of projecting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §3.6 carried an estimate for the next step. It is now a measurement: `fieldMod` / `cFieldMod` were patched behind a throwaway switch to emit the short form, the corpus was re-measured, and the patch was discarded. p256-wallet 958,792 -> 304,463 (pool) -> 179,796 (+ sinking, -81.2%) p384-wallet 1,963,300 -> 463,435 -> 272,584 (-86.1%) ec-primitives 1,332,782 -> 433,880 -> 258,160 (-80.6%) The sound variant captures 89% of the theoretical floor, so the analysis does not need to be clever about subtraction. The two optimizations are also not independent: without pooling the cheap `fieldSub` form pushes the prime twice and p256-wallet gets LARGER (958,792 -> 999,371). Sinking only pays once the prime is a 2-byte pick. The more useful result is what the analysis actually has to prove (new §3.8). The short-reduction variant passes 256 EC oracle assertions — OpenSSL signatures on both curves, ec-on-curve-canonicity, ec-degenerate-add, ec-mul-scalars, p256-p384-scalars, p256-p384-ecdsa-verify. It would have shipped looking green. It is still unsound: - multiply / add / mulconst need only `dividend >= 0`, which unsigned coordinate decoding already gives — ~70% of reductions, trivial analysis; - subtract needs `subtrahend < p`, which decoding 32 unsigned bytes does NOT imply. ecAdd((0, 1), (2^256 - 1, 1)) shipping : ...fffffffdfffff85f sinking : ...0001000003d0 0x1000003d0 = 2^32 + 977 = 2^256 - p Reachable only through the unguarded bare builtins; verifyECDSA_* and onCurve run a canonicity guard first. So Phase 4/5 is a sign lattice plus a `< p` bit that only subtrahends carry — materially smaller than a full modular-domain lattice, and the difference between passing 256 oracle assertions and being correct. Also reorders §6. At 179,796 bytes the split is 69.6% stack-shuffle / 26.7% arithmetic and OP_PICK (x36,683) is the largest opcode: once a reduction costs 3 bytes, ECTracker's own operand shuffling is the bottleneck, so the typed field-element IR moves ahead of Straus/comb. No code changes; the experiment branch was deleted. --- .../script-size-optimizer-results.md | 115 +++++++++++++++--- 1 file changed, 96 insertions(+), 19 deletions(-) diff --git a/docs/experiments/script-size-optimizer-results.md b/docs/experiments/script-size-optimizer-results.md index eb538164..ca780d44 100644 --- a/docs/experiments/script-size-optimizer-results.md +++ b/docs/experiments/script-size-optimizer-results.md @@ -23,6 +23,10 @@ Everything below is measured on the 72 conformance fixtures with **`conformance/tests/p256-wallet`: 958,792 → 304,463 bytes (−68.2 %)**, which is the brief's 959 kB reference implementation. `p384-wallet`: 1,963,300 → 463,435 (−76.4 %). +The next step after this slice has been measured rather than projected: adding reduction +sinking takes `p256-wallet` to **179,796 bytes (−81.2 % from shipping)** and `p384-wallet` to +272,584 (−86.1 %). See §3.7, and §3.8 for the precondition it turns out to need. + Default output is unchanged: all 72 fixtures still reproduce their checked-in `expected-script.hex` byte-for-byte (`packages/runar-compiler/src/__tests__/golden-invariance.test.ts`), and the Go and Rust @@ -140,8 +144,9 @@ results differ only because P-384's prime is a 50-byte push instead of 34. ### 3.6 Can Rúnar approach the ~29.6 kB P-256 result? -Not yet — `p256-wallet` is at 304,463 bytes, 10× the target. But the remaining bytes are now -in exactly two buckets, and neither is mysterious: +Not yet — `p256-wallet` is at 304,463 bytes, 10× the target. But the remaining bytes are in +exactly two buckets, neither is mysterious, and the size of the next step has been **measured** +rather than projected (see §3.7). | category | bytes | share | |---|---:|---:| @@ -160,22 +165,85 @@ OP_2DUP OP_MOD OP_ROT OP_DROP OP_OVER OP_ADD OP_SWAP OP_MOD 8 bytes ``` The six-opcode tail after the first `OP_MOD` exists only because `OP_MOD` takes the sign of -the dividend. For a product of two values already reduced into `[0, p)` the dividend is -non-negative and the tail is dead weight. Knowing that requires modular-domain analysis — -**brief Phase 4/5, and now clearly the single highest-value next step**: +the dividend. Where the dividend is provably non-negative the tail is dead weight. + +### 3.7 Measured ceiling for reduction sinking + +Rather than project the next step, it was measured directly: `fieldMod` / `cFieldMod` were +patched behind a throwaway env switch to emit the short form, the corpus was re-measured, and +the patch was discarded. Two variants: + +- `nonneg` — short reduction where the dividend is provably ≥ 0 (`fieldMul`, `fieldSqr`, + `fieldAdd`, `fieldMulConst`), and for `fieldSub` the cheap `a − b + p` then one `OP_MOD` + (6 bytes instead of 10). This is what a correct analysis could actually emit. +- `all` — short reduction everywhere. Semantically wrong; the absolute floor. + +| fixture | shipping | + pool | **+ pool + sinking** | floor (`all`) | +|---|---:|---:|---:|---:| +| `p256-wallet` | 958,792 | 304,463 | **179,796** (−81.2 %) | 164,302 | +| `p384-wallet` | 1,963,300 | 463,435 | **272,584** (−86.1 %) | 249,410 | +| `ec-primitives` | 1,332,782 | 433,880 | **258,160** (−80.6 %) | 237,229 | +| `ec-unit` | 479,716 | 157,129 | **93,585** (−80.5 %) | 86,576 | + +**The sound variant captures 89 % of the theoretical floor** (124,667 of a possible 140,161 +bytes on `p256-wallet`), so the analysis does not need to be clever about subtraction — the +cheap `+p` form is nearly free. + +Note the pooling and the sinking are complementary, not independent: without pooling, the +cheap `fieldSub` form pushes the prime *twice* and `p256-wallet` gets **larger** (958,792 → +999,371). Sinking only pays once the prime is a 2-byte pick. + +### 3.8 The analysis needed is a sign lattice, not a modular-domain lattice + +The `nonneg` variant passes **256 EC oracle assertions** — OpenSSL signatures on both curves, +`ec-on-curve-canonicity`, `ec-degenerate-add`, `ec-mul-scalars`, `p256-p384-scalars`, +`p256-p384-ecdsa-verify`. It would have shipped looking green. + +It is nonetheless unsound, in a narrow and precisely characterised window: + +- The **multiply / add / mulconst** paths need only *dividend ≥ 0*. That is already implied by + `OP_BIN2NUM` of unsigned coordinate bytes, by products of non-negatives, and by sums of + non-negatives. Roughly 70 % of all reductions qualify under a trivial sign analysis. +- The **subtract** path needs the strictly stronger *subtrahend < p*, and that is NOT implied + by "decoded from 32 unsigned bytes". + +The concrete divergence, found by construction: + +``` +ecAdd((0, 1), (2^256 − 1, 1)) + shipping : fffffffffffffffffffffffffffffffffffffffffffffffffffffffdfffff85f… + sinking : 00000000000000000000000000000000000000000000000001000003d0… + ^ 0x1000003d0 = 2^32 + 977 = 2^256 − p +``` + +It bites only when the subtrahend is non-canonical *and* the minuend is smaller than the gap +between `p` and `2^256` — a ~2^32-wide window out of 2^256, reachable only through the +unguarded bare builtins (`ecAdd`, `p256Add`, `p256Mul` take raw coordinates; +`verifyECDSA_*` and `onCurve` run a canonicity guard first). + +So the requirement for Phase 4/5 is sharper than "modular-domain analysis": a **sign lattice** +plus a **`< p` bit that only subtrahends need**. That is a materially smaller piece of work +than a full domain lattice, and it is the difference between an optimization that passes 256 +oracle assertions and one that is actually correct. + +### 3.9 Revised trajectory ``` -304,463 today - − ~140,000 drop the sign-normalisation tail where the domain proves it dead -≈ 164,000 after lazy/typed modular reduction - − ~80,000 Straus/Shamir: one joint ladder instead of two (brief Phase 9) -≈ 84,000 - − ~50,000 fixed-base comb for u1·G, G being compile-time known (Phases 10–11) -≈ 34,000 ← the reference trajectory's comb stage (34,470 B) +958,792 shipping +179,796 + constant pool + reduction sinking (MEASURED) + − Straus/Shamir: one joint ladder instead of two (Phase 9, estimated) + − fixed-base comb for u1·G, G compile-time known (Phases 10–11, estimated) + ~30,000 ← the reference trajectory's comb stage (34,470 B) ``` -That projection lands on the reference numbers, which is a reason to believe the remaining -gap is algorithmic rather than structural. It is a projection, not a measurement. +Everything above 179,796 is measured; everything below it is still an estimate. + +Note what the measurement does to the ordering. At 179,796 bytes the split is 69.6 % +stack-shuffle / 26.7 % arithmetic, and `OP_PICK` (×36,683) is the single largest opcode. Once +a reduction costs 3 bytes, **`ECTracker`'s own operand shuffling is the bottleneck, not the +reduction**. Straus and comb still help — they cut total operations, so both columns shrink — +but the structural fix underneath is the field-element IR in §4: it would put those 125,119 +bytes of shuffle within reach of a scheduler, which nothing can reach today. --- @@ -258,11 +326,20 @@ Two process notes worth carrying forward: **Do next (highest value first):** -5. **Modular-domain analysis + reduction sinking** (brief Phases 4–5). ~140 kB on - `p256-wallet`, and it is the prerequisite for everything after it. -6. **Straus/Shamir joint ladder**, then a **fixed-base comb** for `u1·G` (Phases 9–11). -7. **Witness-hint modular inverse** (Phase 7) — removes three unrolled Fermat ladders - (382 + 423 + 286 field multiplications per P-256 verify). +5. **Sign analysis + reduction sinking** (brief Phases 4–5). **Measured at 124,667 bytes on + `p256-wallet`** (304,463 → 179,796) — see §3.7. Two facts to carry: *dividend ≥ 0*, which a + trivial sign lattice gives for ~70 % of reductions, and *subtrahend < p*, which only + `fieldSub` needs and which unsigned 32-byte decoding does NOT imply (§3.8). It is also the + prerequisite for everything after it: Straus, comb and lazy accumulation all need to know + which values are already reduced. +6. **A typed field-element IR under the crypto emitters.** After §5 lands, 69.6 % of what + remains is `ECTracker`'s own operand shuffling, which no pass can currently reach. This is + larger than any single item here, and it would let §5, §7 and future work be written once + instead of seven times (§4). +7. **Straus/Shamir joint ladder**, then a **fixed-base comb** for `u1·G` (Phases 9–11). +8. **Witness-hint modular inverse** (Phase 7) — removes three unrolled Fermat ladders + (382 + 423 + 286 field multiplications per P-256 verify). The first item requiring an + explicit soundness argument rather than a translation-validation proof. ---