Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
1cd3d59
uorc: native codec fast path and framed streaming (#105, #106)
claude Nov 9, 2026
82bdbfb
uorc: ncm context-mixing coder and a linked standard kernel basis (WI…
claude Nov 9, 2026
1895ac5
uorc: native codec fast path and framed streaming (#105, #106)
claude Nov 9, 2026
d8c07cf
uorc: optimize the client crate in test builds, lighter debug differe…
claude Nov 9, 2026
e740383
uorc: rustfmt the ulz history test
claude Nov 9, 2026
aaf40f0
client: ncm LexLean sized for the rust-std certificate
claude Nov 9, 2026
408e3f4
uorc model: window frames over the standard basis
claude Nov 9, 2026
e452e68
client: fix window archive model syntax; refresh exact axiom lists
claude Nov 9, 2026
795215b
build: optimize the exported reference crates in test builds
claude Nov 9, 2026
aea9682
generated: re-export the client (ncm, linked basis, window frames)
claude Nov 9, 2026
67bc485
Merge branch 'main' into claude/project-thread-e7b16t
afflom Nov 9, 2026
d101f2b
Merge branch 'main' into claude/project-thread-e7b16t
afflom Nov 9, 2026
8716519
uorc: ncm context-mixing level and the standard kernel basis
claude Nov 9, 2026
b8324ab
Merge remote-tracking branch 'origin/claude/project-thread-e7b16t' in…
claude Nov 9, 2026
f24f0f5
Supersede the earlier ratio-lane history (ported onto #112)
claude Nov 9, 2026
b3ced66
Merge remote-tracking branch 'origin/claude/project-thread-e7b16t' in…
claude Nov 9, 2026
daa639b
uorc: ncm context-mixing level (--best) and the standard kernel basis…
afflom Nov 9, 2026
16a678f
Merge main (#112 squash) into the ratio branch to re-land #115 on main
claude Nov 9, 2026
cc7b9df
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 9, 2026
5610fb1
uorc: native window frames (each frame matches into the two before it)
claude Nov 9, 2026
04ed5b8
Merge remote-tracking branch 'origin/claude/project-thread-e7b16t' in…
claude Nov 9, 2026
5cfda0a
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 10, 2026
edf2926
uorc: ncm v2, 3.9% smaller on Canterbury
claude Nov 10, 2026
ecb0597
ulz: hash4 chain finder with hash3 head (model, mirror, native)
claude Nov 10, 2026
213ce08
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 10, 2026
bc1a5c5
Re-export generated Rust for the hash4 finder; rustfmt ulz.rs
claude Nov 10, 2026
417371c
Merge remote-tracking branch 'origin/claude/project-thread-84bxra' in…
claude Nov 10, 2026
84853ad
Merge the merged window-frames branch history (no content)
claude Nov 10, 2026
542283a
Merge main (#119 squash) into the finder branch
claude Nov 10, 2026
14ec007
Restore the finder export in generated/rust (fix the bad merge resolu…
claude Nov 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 26 additions & 13 deletions client/gen/gen_enc.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
# Generates Uorc/Codec/LzEncode.lex.tex (LexLean source). The generator is a writing aid; the .lex.tex is the source.
# The encoder state is (mem, EncS): one byte table and a record of scalars (see state.py).
# mem holds the probabilities (bytes [0, 16384)), the hash heads (65536 four-byte entries from byte 16384),
# the chain (four-byte entries from byte 278528), and then the output from byte outBase.
# mem holds the probabilities (bytes [0, 16384)), the 3-byte hash table (65536 four-byte entries from byte
# 16384: the latest position of each hash), the 4-byte hash heads (h4size four-byte entries from byte 278528),
# the chain through them (four-byte entries from word chainAt), and then the output from byte outBase.
# h4size is 2^hb with hb the input's bit length clamped to [16, 22], so small inputs keep small tables.
import sys
import os
HERE = os.path.dirname(os.path.abspath(__file__))
Expand All @@ -11,7 +13,8 @@
T = 'Uorc.Codec.Table'; DEC = 'Uorc.Codec.LzDecode'
L = Layout('Enc', ['mem'],
['data:Bytes', 'n:Nat', 'low:Nat', 'rng:Nat', 'cache:Nat', 'csize:Nat', 'outSize:Nat',
'state:Nat', 'rep0:Nat', 'rep1:Nat', 'rep2:Nat', 'rep3:Nat', 'at:Nat', 'outBase:Nat'], {})
'state:Nat', 'rep0:Nat', 'rep1:Nat', 'rep2:Nat', 'rep3:Nat', 'at:Nat', 'outBase:Nat',
'h4div:Nat', 'h4size:Nat', 'chainAt:Nat'], {})
P, upd = L.P, L.upd
defs = []
D = definer(defs, 'LzEncode')
Expand Down Expand Up @@ -52,18 +55,26 @@
D('distEnc', ['e : Enc', 'dist : Nat', 'lenCode : Nat'], 'Enc', de)
# match finding reads the input and the tables without taking the state
D('hashAt', ['data : Bytes', 'i : Nat'], 'Nat', f"(let x : Nat := ((({B('data','i')} * 65536) + ({B('data','(i + 1)')} * 256)) + {B('data','(i + 2)')}); (prim remainder (prim quotient (x * 2654435761) 65536 0 : Nat) 65536 0 : Nat))")
ins = (f"(if ((i + 2) <? {P('e','n')}) then (let k : Nat := (hashAt {P('e','data')} i); (let hv : Nat := ({T}::get 4 {P('e','mem')} (4096 + k)); "
+ upd('e', mem=f"({T}::set 4 ({T}::set 4 {P('e','mem')} (69632 + i) hv) (4096 + k) (i + 1))") + ")) else e)")
D('hash4At', ['data : Bytes', 'i : Nat', 'hdiv : Nat', 'hsz : Nat'], 'Nat',
f"(let x : Nat := ((({B('data','i')} + (256 * {B('data','(i + 1)')})) + (65536 * {B('data','(i + 2)')})) + (16777216 * {B('data','(i + 3)')})); (prim remainder (prim quotient (2654435761 * x) hdiv 0 : Nat) hsz 0 : Nat))")
ins = (f"(let n : Nat := {P('e','n')}; (let data : Bytes := {P('e','data')}; "
f"(let f : Enc := (if ((i + 3) <? n) then (let k : Nat := (69632 + (hash4At data i {P('e','h4div')} {P('e','h4size')})); (let hv : Nat := ({T}::get 4 {P('e','mem')} k); "
+ upd('e', mem=f"({T}::set 4 ({T}::set 4 {P('e','mem')} ({P('e','chainAt')} + i) hv) k (i + 1))") + ")) else e); "
f"(if ((i + 2) <? n) then {upd('f', mem=f'({T}::set 4 {P(chr(102),chr(109)+chr(101)+chr(109))} (4096 + (hashAt data i)) (i + 1))')} else f))))")
D('insert', ['e : Enc', 'i : Nat'], 'Enc', ins)
D('insertRange', ['fuel : Nat', 'e : Enc', 'i : Nat'], 'Enc', "(match fuel\n | Nat.zero => e\n | Nat.succ k => (insertRange k (insert e i) (i + 1)))", 'fuel')
D('matchLen', ['fuel : Nat', 'data : Bytes', 'a : Nat', 'b : Nat', 'l : Nat'], 'Nat', f"(match fuel\n | Nat.zero => l\n | Nat.succ k => (if ({B('data','(a + l)')} =? {B('data','(b + l)')}) then (matchLen k data a b (l + 1)) else l))", 'fuel')
fl = (f"(match fuel\n | Nat.zero => (best, dist)\n | Nat.succ k => (if (candidate =? 0) then (best, dist) else (let j : Nat := (prim subtract candidate 1 : Nat); (let next : Nat := ({T}::get 4 chain (69632 + j)); "
f"(if ({B('data','(j + best)')} =? {B('data','(i + best)')}) then (let l : Nat := (matchLen lim data j i 0); (let nb : Nat := (if (best <? l) then l else best); (let nd : Nat := (if (best <? l) then (prim subtract i (j + 1) : Nat) else dist); (if (64 <=? l) then (nb, nd) else (findLoop k data chain i lim next nb nd))))) "
f"else (findLoop k data chain i lim next best dist))))))")
D('findLoop', ['fuel : Nat', 'data : Bytes', 'chain : Bytes', 'i : Nat', 'lim : Nat', 'candidate : Nat', 'best : Nat', 'dist : Nat'], '(Nat * Nat)', fl, 'fuel')
fl = (f"(match fuel\n | Nat.zero => (best, dist)\n | Nat.succ k => (if (candidate =? 0) then (best, dist) else (let j : Nat := (prim subtract candidate 1 : Nat); (let next : Nat := ({T}::get 4 chain (cb + j)); "
f"(if ({B('data','(j + best)')} =? {B('data','(i + best)')}) then (let l : Nat := (matchLen lim data j i 0); (let nb : Nat := (if (best <? l) then l else best); (let nd : Nat := (if (best <? l) then (prim subtract i (j + 1) : Nat) else dist); (if (64 <=? l) then (nb, nd) else (findLoop k data chain cb i lim next nb nd))))) "
f"else (findLoop k data chain cb i lim next best dist))))))")
D('findLoop', ['fuel : Nat', 'data : Bytes', 'chain : Bytes', 'cb : Nat', 'i : Nat', 'lim : Nat', 'candidate : Nat', 'best : Nat', 'dist : Nat'], '(Nat * Nat)', fl, 'fuel')
D('limitAt', ['n : Nat', 'i : Nat'], 'Nat', "(let left : Nat := (prim subtract n i : Nat); (if (left <? 273) then left else 273))")
D('find', ['e : Enc', 'i : Nat'], '(Nat * Nat)',
f"(let n : Nat := {P('e','n')}; (if (n <=? (i + 2)) then (0, 0) else (let data : Bytes := {P('e','data')}; (let h : Nat := ({T}::get 4 {P('e','mem')} (4096 + (hashAt data i))); (findLoop 48 data {P('e','mem')} i (limitAt n i) h 0 0)))))")
# the latest position with the same 3-byte hash gives the short matches; the 4-byte chain, the long ones
fd = (f"(let n : Nat := {P('e','n')}; (if (n <=? (i + 2)) then (0, 0) else (let data : Bytes := {P('e','data')}; (let mem : Bytes := {P('e','mem')}; (let lim : Nat := (limitAt n i); "
f"(let c3 : Nat := ({T}::get 4 mem (4096 + (hashAt data i))); "
f"(let s : (Nat * Nat) := (if (c3 =? 0) then (0, 0) else (let j : Nat := (prim subtract c3 1 : Nat); (let l : Nat := (matchLen lim data j i 0); (if (2 <=? l) then (l, (prim subtract i (j + 1) : Nat)) else (0, 0))))); "
f"(if ((n <=? (i + 3)) || (64 <=? (first s))) then s else (let h : Nat := ({T}::get 4 mem (69632 + (hash4At data i {P('e','h4div')} {P('e','h4size')}))); (findLoop 48 data mem {P('e','chainAt')} i lim h (first s) (second s)))))))))))")
D('find', ['e : Enc', 'i : Nat'], '(Nat * Nat)', fd)
D('repLen', ['data : Bytes', 'i : Nat', 'lim : Nat', 'r : Nat'], 'Nat', "(if (i <? (r + 1)) then 0 else (matchLen lim data (prim subtract i (r + 1) : Nat) i 0))")
br = (f"(let data : Bytes := {P('e','data')}; (let l0 : Nat := (repLen data i lim {P('e','rep0')}); (let l1 : Nat := (repLen data i lim {P('e','rep1')}); (let l2 : Nat := (repLen data i lim {P('e','rep2')}); (let l3 : Nat := (repLen data i lim {P('e','rep3')}); "
"(let a : (Nat * Nat) := (if (l0 <? l1) then (l1, 1) else (l0, 0)); (let b : (Nat * Nat) := (if ((first a) <? l2) then (l2, 2) else a); (if ((first b) <? l3) then (l3, 3) else b))))))))")
Expand Down Expand Up @@ -92,8 +103,10 @@
D('encodeLoop', ['fuel : Nat', 'e : Enc'], 'Enc', f"(match fuel\n | Nat.zero => e\n | Nat.succ k => (let i : Nat := {P('e','at')}; (let n : Nat := {P('e','n')}; (if (n <=? i) then e else (encodeLoop k (step e))))))", 'fuel')
D('header', ['fuel : Nat', 'v : Nat'], 'Bytes', f"(match fuel\n | Nat.zero => 0x\n | Nat.succ k => (prim append ({DEC}::singleton (prim remainder v 256 0 : Nat)) (header k (prim quotient v 256 0 : Nat)) : Bytes))", 'fuel')
start = (f"(let n : Nat := (prim length data : Nat); (let od : Nat := ({DEC}::depthFor 64 ((2 * n) + 64) 0); (let cd : Nat := ({DEC}::depthFor 64 (n + 1) 0); "
+ L.make({'mem': f"(prim append (prim append (prim append ({T}::fill 2 13 1024) ({T}::fill 4 16 0) : Bytes) ({T}::fill 4 cd 0) : Bytes) ({T}::fill 1 od 0) : Bytes)"},
dict(data='data', n='n', low='0', rng='4294967295', cache='0', csize='1', outSize='0', state='0', rep0='0', rep1='0', rep2='0', rep3='0', at='0', outBase=f'(278528 + (4 * {pow2(chr(99)+chr(100))}))')) + ")))")
f"(let hb : Nat := (if (cd <? 16) then 16 else (if (22 <? cd) then 22 else cd)); (let h4size : Nat := {pow2('hb')}; "
+ L.make({'mem': f"(prim append (prim append (prim append (prim append ({T}::fill 2 13 1024) ({T}::fill 4 16 0) : Bytes) ({T}::fill 4 hb 0) : Bytes) ({T}::fill 4 cd 0) : Bytes) ({T}::fill 1 od 0) : Bytes)"},
dict(data='data', n='n', low='0', rng='4294967295', cache='0', csize='1', outSize='0', state='0', rep0='0', rep1='0', rep2='0', rep3='0', at='0',
outBase=f'((278528 + (4 * h4size)) + (4 * {pow2(chr(99)+chr(100))}))', h4div=pow2('(prim subtract 32 hb : Nat)'), h4size='h4size', chainAt='(69632 + h4size)')) + ")))))")
D('start', ['data : Bytes'], 'Enc', start)
# encode `input` against the window `history`: the match finder starts holding every history position,
# the coder starts at the first input byte, and only the input is emitted
Expand Down
18 changes: 15 additions & 3 deletions client/gen/ulz.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,18 +134,30 @@ def compress_with(hist, data, depth=48, nice=64):
# encode `data` against the window `hist`: matches may reach into it; only `data` is emitted
hl=len(hist); m_in=len(data); data=bytes(hist)+bytes(data)
e=Enc(); m=model(); L=lenmodel(); R=lenmodel()
n=len(data); HB=16; head=[-1]*(1<<HB); chain=[-1]*n
n=len(data); HB=16; head3=[-1]*(1<<HB); chain=[-1]*n
cd=0
while (1<<cd)<n+1: cd+=1
H4=min(22,max(16,cd)); head=[-1]*(1<<H4)
def h(i): return ((data[i]<<16|data[i+1]<<8|data[i+2])*2654435761>>(32-HB))&((1<<HB)-1)
def h4(i): return ((int.from_bytes(data[i:i+4],'little')*2654435761)>>(32-H4))&((1<<H4)-1)
def ins(i):
if i+2<n: k=h(i); chain[i]=head[k]; head[k]=i
if i+3<n: k=h4(i); chain[i]=head[k]; head[k]=i
if i+2<n: head3[h(i)]=i
def mlen(a,b,lim):
l=0
while l<lim and data[a+l]==data[b+l]: l+=1
return l
def find(i):
# the latest position with the same 3-byte hash gives the short matches; the 4-byte chain, the long ones
best=0; bd=0
if i+2>=n: return 0,0
j=head[h(i)]; c=depth; lim=min(MAXM,n-i)
lim=min(MAXM,n-i)
j=head3[h(i)]
if j>=0:
l=mlen(j,i,lim)
if l>=2: best=l; bd=i-j-1
if i+3>=n or best>=nice: return best,bd
j=head[h4(i)]; c=depth
while j>=0 and c>0:
if (data[j+best] if j+best<n else 0)==(data[i+best] if i+best<n else 0): # an out-of-range read is 0, as in LexLean
l=mlen(j,i,lim)
Expand Down
Loading
Loading