Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
f08f89e
ncm: profiles (context masks) with compact mixer rows; one-line conte…
claude Nov 10, 2026
2564bde
ncm v4 features: short match model and repeat-distance model, profile…
claude Nov 10, 2026
8859ca0
wip: profile plumbing in the generators
claude Nov 10, 2026
08002b5
Merge lane 1's ncm v4 features (short match, repeat model) onto the p…
claude Nov 10, 2026
27503e4
ncm v4: match-state mixer selection as its own profile bit; new input…
claude Nov 10, 2026
956cdca
wip: lane 1 F_SEL + zero-start weights; v0.3 era, profiled entries in…
claude Nov 10, 2026
5c26010
uorc: per-frame strategy portfolio with rule30 weights learned per da…
claude Nov 10, 2026
fd4727e
uorc: profiled ncm entries over an appended library; v0.3 archives ke…
claude Nov 10, 2026
477a9ba
Merge lane 2's profiled ncm entries; the portfolio writes them throug…
claude Nov 10, 2026
ded207f
wip: archive tests for earlier releases and profiled entries
claude Nov 10, 2026
dc3e0bf
Merge remote-tracking branch 'origin/claude/project-thread-2twygm' in…
claude Nov 10, 2026
3a67cef
uorc: code a frame's two scheduled strategies side by side
claude Nov 10, 2026
8a5f0a5
uorc: the v0.3 profile writes the v0.3 archive; family binder renamed…
claude Nov 10, 2026
2a4dcc9
Merge lane 1's parallel strategy coding
claude Nov 10, 2026
c122eb6
uorc: reference frames find frames labelled over the v0.3 library
claude Nov 10, 2026
83d2220
uorc: exact axiom policies for the profiled and v0.3 entries
claude Nov 10, 2026
49636ce
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 10, 2026
15381d8
uorc: the encoder learns each frame's context mask
claude Nov 10, 2026
772d874
uorc: portfolio outcomes generator and a threads-determinism test
claude Nov 10, 2026
0ace928
ncm: kernels_agree calls the encoder with its learn flag
claude Nov 10, 2026
c4fd13f
Merge remote-tracking branch 'origin/claude/project-thread-2twygm' in…
claude Nov 10, 2026
4094711
ncm: a WebAssembly SIMD128 kernel for the mixers, byte-identical
claude Nov 10, 2026
cafd25e
uorc: a strategy stops once its archive cannot be chosen
claude Nov 10, 2026
8b9e91e
Merge remote-tracking branch 'origin/claude/project-thread-2twygm' in…
claude Nov 10, 2026
84ba3dc
generated: export the v0.4 client (profiled entries, ncm v4, learned …
claude Nov 11, 2026
605d9fe
Merge main (factory, neuromorphic standard, repeats at any offset) in…
claude Nov 11, 2026
329c58a
uorc: released v0.2.0 and v0.3.0 streams decode, as a test
claude Nov 11, 2026
6f9a820
uorc: cargo fmt
claude Nov 11, 2026
89fc486
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 11, 2026
db6eb56
Merge main (per-leg factory closures) into lane 2; workspace version …
claude Nov 11, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion .cargo/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,7 @@ vv = "run --package xtask -- validate"
# The exported uorc code recurses structurally over archives and byte ropes;
# the default 1 MiB WebAssembly stack is too small for it. 2 KiB inputs run in
# 16 MiB, and 64 MiB leaves room for deeper inputs.
# simd128 lets ncm's mixers run on the WebAssembly SIMD kernel (every browser
# the page supports has it).
[target.wasm32-unknown-unknown]
rustflags = ["-C", "link-arg=-zstack-size=67108864"]
rustflags = ["-C", "link-arg=-zstack-size=67108864", "-C", "target-feature=+simd128"]
10 changes: 5 additions & 5 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ members = ["crates/*", "xtask"]
exclude = ["generated/rust"]

[workspace.package]
version = "0.3.0"
version = "0.4.0"
edition = "2021"
# Raise this when something here needs a newer compiler; nothing does yet.
rust-version = "1.89"
Expand All @@ -19,11 +19,11 @@ keywords = []
categories = []

[workspace.dependencies]
repo-model = { version = "0.3.0", path = "crates/model" }
repo-model = { version = "0.4.0", path = "crates/model" }
# A `path` dependency with no `version` is a wildcard requirement, and
# `cargo deny` refuses those (`wildcards = "deny"`). Every internal dependency
# carries both.
repo-conformance = { version = "0.3.0", path = "crates/conformance" }
repo-conformance = { version = "0.4.0", path = "crates/conformance" }

[workspace.lints.rust]
missing_docs = "deny"
Expand Down
32 changes: 31 additions & 1 deletion client/gen/axioms.json
Original file line number Diff line number Diff line change
Expand Up @@ -82,5 +82,35 @@
"Archive.v02Window2Suffix": [],
"Archive.v02WindowPrefix": [],
"Archive.v02WindowSuffix": [],
"Archive.v02Root": []
"Archive.v02Root": [],
"Archive.v03Identity": [],
"Archive.v03Blocks": [],
"Archive.v03Library": [],
"Archive.v03UlzSuffix": [],
"Archive.v03NcmSuffix": [],
"Archive.v03NcmxSuffix": [],
"Archive.profilePrefix": [],
"Archive.profileTail": [],
"Archive.profiledBody": [],
"Archive.profileWord": [],
"Archive.v03Root": [],
"Archive.v03WindowSuffix": [],
"Archive.profile0Prefix": [],
"Archive.profile0Tail": [],
"Archive.profile0XPrefix": [],
"Archive.profile0XTail": [],
"Archive.profile1Prefix": [],
"Archive.profile1Tail": [],
"Archive.profile1XPrefix": [],
"Archive.profile1XTail": [],
"Archive.profile2Prefix": [],
"Archive.profile2Tail": [],
"Archive.profile2XPrefix": [],
"Archive.profile2XTail": [],
"Archive.v03Window1Suffix": [],
"Archive.v03Window1NcmSuffix": [],
"Archive.v03Window1XNcmSuffix": [],
"Archive.v03Window2Suffix": [],
"Archive.v03Window2NcmSuffix": [],
"Archive.v03Window2XNcmSuffix": []
}
103 changes: 98 additions & 5 deletions client/gen/gen_archive.py

Large diffs are not rendered by default.

100 changes: 92 additions & 8 deletions client/gen/gen_basis.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,27 @@ def libraries():
nwith = ncmgraph.window_library(g)
_, x86 = x86graph.library(g)
g.x86 = x86
runs = {'ulz': urun, 'ncm': nrun, 'ulz1': uwith, 'ulz2': uwith, 'ncm1': nwith, 'ncm2': nwith}
# every block above is the v0.3 library, unchanged; the profiled coder (ncm_model) follows
g.v03_blocks = len(g.blocks)
nprof = ncmgraph.profile_library(g)
runs = {'ulz': urun, 'ncm': nrun, 'ulz1': uwith, 'ulz2': uwith, 'ncm1': nwith, 'ncm2': nwith,
'ncmp': nprof, 'ncmp1': nprof, 'ncmp2': nprof}
runs.update({c + 'x': r for c, r in list(runs.items()) if c.startswith('ncm')})
return g, runs


# Window entries: roots 1..K of the basis are the outputs of the K preceding frames, oldest first.
WINDOWS = {'ulz1': 1, 'ulz2': 2, 'ncm1': 1, 'ncm2': 2}
WINDOWS = {'ulz1': 1, 'ulz2': 2, 'ncm1': 1, 'ncm2': 2, 'ncmp1': 1, 'ncmp2': 2}
WINDOWS.update({c + 'x': k for c, k in list(WINDOWS.items()) if c.startswith('ncm')})
# The client compresses with ncm only; the ulz entries stay so earlier archives keep their meaning.
# Codecs ending in x code the x86-filtered bytes (x86_model): the entry undoes the filter on the
# decoder's output, and a window entry filters the history first, the frame placed after it.
CODECS = ['ulz', 'ncm', 'ulz1', 'ulz2', 'ncm1', 'ncm2', 'ncmx', 'ncm1x', 'ncm2x']
# Profiled codecs (ncmp*): the entry applies ncmRunProfile to the payload, the history (empty without a window)
# and the frame's profile word, a natural constant right after the payload: the template is
# prefix ++ uleb(len) ++ payload ++ 00 ++ uleb(profile) ++ tail.
PCODECS = ['ncmp', 'ncmp1', 'ncmp2', 'ncmpx', 'ncmp1x', 'ncmp2x']
PROFILE_MARK = 0x7ffff


G, RUN = libraries()
Expand All @@ -66,9 +75,34 @@ def _hex(name):
assert V02_ID == wire.chain(TAG + wire.uleb(1) + wire.uleb(len(V02_LIB)) + V02_LIB)
assert V02_LIB == wire.uleb(G.v02_blocks) + b''.join(block_bytes(b) for b in G.blocks[:G.v02_blocks]), 'the v0.2 blocks changed'

# The v0.3 basis (client/gen/v03): its library is this one's first v03_blocks blocks, byte for byte, so v0.3
# archives keep their meaning too. The client accepts its identity beside STD_ID.
def _hex3(name):
return bytes.fromhex(open(os.path.join(HERE, 'v03', name)).read().replace('\n', ''))
V03_LIB = _hex3('basis-library.hex')
V03_ID = _hex3('basis-id.hex')
assert V03_ID == wire.chain(TAG + wire.uleb(1) + wire.uleb(len(V03_LIB)) + V03_LIB)
assert V03_LIB == wire.uleb(G.v03_blocks) + b''.join(block_bytes(b) for b in G.blocks[:G.v03_blocks]), 'the v0.3 blocks changed'
# a v0.3 entry: its frozen suffix with the current prefix (prefixes reference no block)
V03_TEMPLATES = None

def entry(g, run, payload, k=0, x86=False):

def entry(g, run, payload, k=0, x86=False, profile=None):
e = g.block('entry', [], S)
if profile is not None:
pl = e.seq(payload)
pf = e.nat(profile)
history = e.seq(b'')
if k > 0:
history = e.basis(1)
for r in range(2, k + 1):
history = e.sconcat(history, e.basis(r))
if not x86:
e.returns(e.apply(run, [pl, history, pf]))
return e
out = e.apply(run, [pl, e.apply(g.x86, [history, e.nat(0), e.nat(0)]), pf])
e.returns(e.apply(g.x86, [out, e.nat(1), e.slen(history)]))
return e
if k == 0:
out = e.apply(run, [e.seq(payload)])
e.returns(e.apply(g.x86, [out, e.nat(1), e.nat(0)]) if x86 else out)
Expand Down Expand Up @@ -104,6 +138,22 @@ def template(codec):


TEMPLATES = {c: template(c) for c in CODECS}
V03_TEMPLATES = {c: (TEMPLATES[c][0], _hex3(f'linked-{c}-suffix.hex')) for c in CODECS}


def profile_template(codec):
"""(prefix, tail) of a profiled codec: its body is prefix ++ uleb(len) ++ payload ++ 00 ++ uleb(profile) ++ tail."""
e = entry(G, RUN[codec], PH, WINDOWS.get(codec, 0), codec.endswith('x'), PROFILE_MARK)
eb = block_bytes(e)
G.blocks.pop()
body = wire.uleb(1) + eb + wire.uleb(0)
mark = bytes([3, 2]) + PH + bytes([0]) + wire.uleb(PROFILE_MARK)
assert body.count(mark) == 1, codec
cut = body.index(mark)
return body[:cut + 1], body[cut + len(mark):]


PTEMPLATES = {c: profile_template(c) for c in PCODECS}


def reference_body():
Expand All @@ -120,7 +170,10 @@ def reference_body():
REF_BODY = reference_body()


def linked_body(codec, payload):
def linked_body(codec, payload, profile=None):
if profile is not None:
pre, tail = PTEMPLATES[codec]
return pre + wire.uleb(len(payload)) + payload + bytes([0]) + wire.uleb(profile) + tail
pre, suf = TEMPLATES[codec]
return pre + wire.uleb(len(payload)) + payload + suf

Expand All @@ -131,8 +184,8 @@ def materialize(body):
return wire.uleb(L + 1) + LIBB + body[1:-1] + wire.uleb(L)


def linked_archive(codec, n, payload, window=()):
body = linked_body(codec, payload)
def linked_archive(codec, n, payload, window=(), profile=None):
body = linked_body(codec, payload, profile)
ident = basis_id(window) if window else STD_ID
pre = b'UORC' + bytes([2, 1, 1, 1]) + wire.uleb(1) + wire.uleb(n) + wire.uleb(len(body)) + ident + body
return wire.sealed(pre)
Expand All @@ -157,15 +210,28 @@ def write_hex(name, data):
write_hex(f'linked-{c}-prefix.hex', pre)
write_hex(f'linked-{c}-suffix.hex', suf)
write_hex('linked-ref.hex', REF_BODY)
write_hex('basis-v03-id.hex', V03_ID)
write_hex('basis-v03-library.hex', V03_LIB)
# v0.3 entries: the same prefixes; their suffixes name the run blocks at their v0.3 (relative) distance
for c in CODECS:
assert TEMPLATES[c][0] == V03_TEMPLATES[c][0], c
write_hex(f'v03-linked-{c}-suffix.hex', V03_TEMPLATES[c][1])
for c, (pre, tail) in PTEMPLATES.items():
write_hex(f'linked-{c}-prefix.hex', pre)
write_hex(f'linked-{c}-tail.hex', tail)
print('library blocks', L, 'bytes', len(LIB), 'identity', STD_ID.hex())
for c, (pre, suf) in TEMPLATES.items():
print(c, 'linked prefix', pre.hex(), 'suffix', suf.hex())
for c, (pre, tail) in PTEMPLATES.items():
print(c, 'profiled prefix', pre.hex(), 'tail', tail.hex())
if '--check' in sys.argv:
import ulz, ncm_model, x86_model
text = b'the standard basis carries the kernels; archives name it by identity. ' * 3
text = text[:100] + bytes([0xE8, 0x10, 0, 0, 0, 0xE8, 0xF0, 0xFF, 0xFF, 0xFF]) + text[100:]
xf = lambda d, base=0: x86_model.py_filter(d, 0, base)
for c, enc in (('ulz', ulz.compress), ('ncm', ncm_model.py_encode), ('ncmx', lambda d: ncm_model.py_encode(xf(d)))):
V03 = ncm_model.V03
for c, enc in (('ulz', ulz.compress), ('ncm', lambda d: ncm_model.py_encode(d, b'', V03)),
('ncmx', lambda d: ncm_model.py_encode(xf(d), b'', V03))):
payload = enc(text)
# the materialized graph is the libraries plus the entry, exactly as built structurally
g, runs = libraries()
Expand All @@ -187,8 +253,10 @@ def write_hex(name, data):
frames = [b'window frame one: the standard basis carries the kernels. ' * 4 + call,
b'window frame two: archives name the kernels by identity. ' * 4 + call,
b'window frame three: the standard basis carries the kernels; archives name it by identity. ' * 2 + call]
wenc = {'ulz': ulz.compress_with, 'ncm': lambda h, d: ncm_model.py_encode(d, h)}
wenc = {'ulz': ulz.compress_with, 'ncm': lambda h, d: ncm_model.py_encode(d, h, V03)}
for c, k in WINDOWS.items():
if c not in CODECS:
continue
window = frames[2 - k:2]
hist = b''.join(window)
if c.endswith('x'):
Expand All @@ -202,3 +270,19 @@ def write_hex(name, data):
assert out == frames[2], (c, out[:60])
alone = wenc[c[:3]](b'', frames[2]) # the window's gain
print(c, 'window payload', len(payload), 'vs alone', len(alone), '; decodes on the mirror')
# profiled entries: the payload coded under the profile word the entry carries
for c in PCODECS:
k = WINDOWS.get(c, 0)
window = frames[2 - k:2]
hist = b''.join(window)
for prof in (ncm_model.FULL, 0x1059e | ncm_model.F_REP):
if c.endswith('x'):
payload = ncm_model.py_encode(xf(frames[2], len(hist)), xf(hist), prof)
else:
payload = ncm_model.py_encode(frames[2], hist, prof)
g, runs = libraries()
e = entry(g, runs[c], payload, k, c.endswith('x'), prof)
assert materialize(linked_body(c, payload, prof)) == wire.graph_bytes(g, e), c
out = evaluate(g, e, basis=[LIB] + window)
assert out == frames[2], (c, hex(prof), out[:60])
print(c, hex(prof), 'profiled payload', len(payload), '; decodes on the mirror')
30 changes: 22 additions & 8 deletions client/gen/gen_ncm.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ def tables(L, n, extra, history='0x'):
# Every table's depth names n: the certificate kernel evaluates whatever is closed, and a closed state
# would have it run the model's whole setup (thousands of table writes) at check time.
t = {'glob': f'(zeros (8 + (prim multiply 0 {n} : Nat)))', 'mem': f'(zeros (22 + (prim multiply 0 {n} : Nat)))', 'T': f'({TB}::fill 3 (tbFor {n}) 0)', 'MT': f'({TB}::fill 4 (mbFor (tbFor {n})) 0)',
'H': f'(prim append {history} (zeros (depthFor 40 {n} 0)) : Bytes)', 'A2': f'({TB}::fill 2 (22 + (prim multiply 0 {n} : Nat)) 0)'}
'H': f'(prim append {history} (zeros (depthFor 40 {n} 0)) : Bytes)', 'A2': f'({TB}::fill 2 (22 + (prim multiply 0 {n} : Nat)) 0)',
# the short match table, and the short match and repeat models' 64 and 128 slots
'M2T': f'({TB}::fill 4 (mbFor (tbFor {n})) 0)', 'M2S': f'({TB}::fill 3 (6 + (prim multiply 0 {n} : Nat)) 0)',
'RS': f'({TB}::fill 3 (7 + (prim multiply 0 {n} : Nat)) 0)'}
t.update(extra)
return t

Expand All @@ -34,20 +37,24 @@ def scalars(L, inp):


# decoder
dec = irlex.Lex(ncm_model.P, 'NcmDecode', ['decodeWith'], ['T', 'H', 'MT', 'A2'])
BIG = ['T', 'H', 'MT', 'A2', 'M2T', 'M2S', 'RS']
V03 = str(ncm_model.V03)
dec = irlex.Lex(ncm_model.P, 'NcmDecode', ['decodeProfile'], BIG)
sizing(dec.D)
n = ("(let n : Nat := (((prim multiply 16777216 (getAt 1 input 0) : Nat) + (prim multiply 65536 (getAt 1 input 1) : Nat)) + "
"((prim multiply 256 (getAt 1 input 2) : Nat) + (getAt 1 input 3))); ")
# against a window: H starts holding the history (h bytes), and only the n coded bytes after it are returned
body = (n + "(let h : Nat := (prim length history : Nat); "
f"(let s0 : {dec.St} := {dec.L.make(tables(dec.L, 'n', {}, 'history'), scalars(dec.L, 'input'))}; "
f"(let s1 : {dec.St} := (decodeWith s0 h); (match (prim slice {dec.L.P('s1', 'H')} h n : (Option Bytes))\n | Option.none => 0x\n | Option.some b => b)))))")
dec.D('runWith', ['history : Bytes', 'input : Bytes'], 'Bytes', f"(if ((prim length input : Nat) <? 4) then 0x else {body})")
f"(let s1 : {dec.St} := (decodeProfile s0 h profile); (match (prim slice {dec.L.P('s1', 'H')} h n : (Option Bytes))\n | Option.none => 0x\n | Option.some b => b)))))")
# the decode under a profile word (ncm_model: bits 0-16 contexts, then the match features); V03 is the v0.3 coder
dec.D('runProfile', ['history : Bytes', 'input : Bytes', 'profile : Nat'], 'Bytes', f"(if ((prim length input : Nat) <? 4) then 0x else {body})")
dec.D('runWith', ['history : Bytes', 'input : Bytes'], 'Bytes', f"(runProfile history input {V03})")
dec.D('run', ['input : Bytes'], 'Bytes', "(runWith 0x input)")
open(os.path.join(OUTDIR, 'NcmDecode.lex.tex'), 'w').write(dec.source([TB]))

# encoder
enc = irlex.Lex(ncm_model.P, 'NcmEncode', ['encodeWith'], ['T', 'H', 'MT', 'A2', 'OUT'])
enc = irlex.Lex(ncm_model.P, 'NcmEncode', ['encodeProfile', 'encodeLearn'], BIG + ['OUT'])
sizing(enc.D)
enc.D('start', ['history : Bytes', 'input : Bytes', 'n : Nat', 'cap : Nat'], enc.St,
enc.L.make(tables(enc.L, 'n', {'OUT': '(zeros (depthFor 40 cap 0))'}, 'history'), scalars(enc.L, 'input')))
Expand All @@ -56,9 +63,16 @@ def scalars(L, inp):
f"(match (prim slice {enc.L.P('s1', 'OUT')} 0 {enc.gread('s1', 'op')} : (Option Bytes))\n | Option.none => 0x\n"
f" | Option.some b => (prim append head b : Bytes))))")
body = ("(let n : Nat := (prim length input : Nat); (let cap : Nat := ((prim multiply 2 n : Nat) + 64); "
"(finish (encodeWith (start history input n cap) n cap (prim length history : Nat)) n)))")
# encode `input` against the window `history`: the match model starts holding every history position
enc.D('runWith', ['history : Bytes', 'input : Bytes'], 'Bytes', f"(if ((prim length input : Nat) <? 4294967296) then {body} else 0x)")
"(finish (encodeProfile (start history input n cap) n cap (prim length history : Nat) profile) n)))")
# encode `input` against the window `history` under a profile word: the match model starts holding every
# history position
enc.D('runProfile', ['history : Bytes', 'input : Bytes', 'profile : Nat'], 'Bytes', f"(if ((prim length input : Nat) <? 4294967296) then {body} else 0x)")
enc.D('runWith', ['history : Bytes', 'input : Bytes'], 'Bytes', f"(runProfile history input {V03})")
# the profile the encoder learns coding `input` under `profile`: without the contexts that saved nothing
learn = ("(let n : Nat := (prim length input : Nat); (let cap : Nat := ((prim multiply 2 n : Nat) + 64); "
f"(let s1 : {enc.St} := (encodeLearn (start history input n cap) n cap (prim length history : Nat) profile); "
f"(if (0 <? {enc.gread('s1', 'ovf')}) then profile else {enc.gread('s1', 'lmask')}))))")
enc.D('learnProfile', ['history : Bytes', 'input : Bytes', 'profile : Nat'], 'Nat', f"(if ((prim length input : Nat) <? 4294967296) then {learn} else profile)")
enc.D('run', ['input : Bytes'], 'Bytes', "(runWith 0x input)")
open(os.path.join(OUTDIR, 'NcmEncode.lex.tex'), 'w').write(enc.source([TB]))
print('ok')
Loading
Loading