Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
71e7530
Prism model: ncm's plasticity as the neuromorphic standard; Pages clo…
claude Nov 10, 2026
ce21a83
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 10, 2026
4d1e0f9
Pages client: Prism product model of the page, generated from it, wit…
claude Nov 10, 2026
9caee8f
Pages client acceptance is a factory obligation keyed on its own inputs
claude Nov 10, 2026
e0e2e30
pages-client: read the single-obligation route outputs
claude Nov 10, 2026
02a0fc0
Merge main (#129 squash, tree identical to the stacked base)
claude Nov 10, 2026
8b7b037
Merge main (#130, repeats at any offset) into the pages client
claude Nov 11, 2026
cbe2df8
Pages client: folders stream through the frame operations
claude Nov 11, 2026
0399607
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 11, 2026
cccf80f
Page acceptance: all-zero frames are reference frames again (#134)
claude Nov 11, 2026
aca17b9
Merge main (#131 per-leg closures, #134) into the pages client
claude Nov 11, 2026
69e3e0d
factory: the formal set holds web/index.html, a projection of the pag…
claude Nov 11, 2026
0ce33f1
Merge remote-tracking branch 'origin/main' into claude/project-thread…
claude Nov 11, 2026
2df0067
Page acceptance: the page restores the released v0.2.0 and v0.3.0 arc…
claude Nov 11, 2026
208149f
template-check: report /tmp use before PrismPM's native regeneration
claude Nov 11, 2026
d7d1ea9
template-check: run PrismPM's template check before the xtask audit
claude Nov 11, 2026
60c4aef
template-check: report /tmp, tree sizes and cargo environment before …
claude Nov 11, 2026
8032234
template-check: build PrismPM's native regeneration in the workspace …
claude Nov 11, 2026
e6399ae
model-write: build into the target the template check names, not besi…
claude Nov 11, 2026
7c7aa57
template-check: the staged regeneration builds in a target directory …
claude Nov 11, 2026
8c8e3df
uorc: code a planned frame one strategy at a time, with a progress tick
claude Nov 11, 2026
86aaa9f
Pages: a worker pool on the visitor's cores, progress within a frame
claude Nov 11, 2026
a5a05d4
ncm: decode a window frame without copying the window, and keep only …
claude Nov 11, 2026
96998f7
Merge the merged #132 branch history (its tree is in main as 7452a14)
claude Nov 11, 2026
dc3b5f9
Pages: bound decode memory across frames, and keep the progress bar m…
claude Nov 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
280 changes: 279 additions & 1 deletion crates/uorc-wasm/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
//! WebAssembly exports of the uorc client, with no unsafe code and no bindgen.
//! WebAssembly exports of the uorc client, with no bindgen; the only unsafe
//! code is the call of the page's progress import.
//!
//! The page asks for an input buffer of a given size, writes the bytes into
//! wasm memory at the returned offset, calls an operation, then reads the
Expand Down Expand Up @@ -285,6 +286,186 @@ pub extern "C" fn decompress_tree() -> i32 {
}))
}

/// Little-endian reader over a request in the input buffer.
struct Fields<'a> {
bytes: &'a [u8],
at: usize,
}

impl<'a> Fields<'a> {
fn take(&mut self, n: usize) -> Result<&'a [u8], uorc::Error> {
let end = self.at.checked_add(n).ok_or(uorc::Error::Decode)?;
let out = self.bytes.get(self.at..end).ok_or(uorc::Error::Decode)?;
self.at = end;
Ok(out)
}
fn u32(&mut self) -> Result<usize, uorc::Error> {
Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4 bytes")) as usize)
}
fn u64(&mut self) -> Result<u64, uorc::Error> {
Ok(u64::from_le_bytes(
self.take(8)?.try_into().expect("8 bytes"),
))
}
fn len(&mut self) -> Result<usize, uorc::Error> {
usize::try_from(self.u64()?).map_err(|_| uorc::Error::Decode)
}
fn pieces(&mut self) -> Result<Vec<u64>, uorc::Error> {
let n = self.u32()?;
(0..n).map(|_| self.u64()).collect()
}
fn end(&self) -> Result<(), uorc::Error> {
if self.at == self.bytes.len() {
Ok(())
} else {
Err(uorc::Error::Decode)
}
}
}

fn plan_records(planned: Vec<uorc::Planned>) -> Vec<u8> {
let mut out = (planned.len() as u32).to_le_bytes().to_vec();
for frame in planned {
match frame {
uorc::Planned::Done { length, archive } => {
out.push(0);
out.extend(length.to_le_bytes());
out.extend((archive.len() as u64).to_le_bytes());
out.extend(archive);
}
uorc::Planned::Code {
bytes,
pieces,
window,
} => {
out.push(1);
out.extend((bytes.len() as u64).to_le_bytes());
out.extend((pieces.len() as u32).to_le_bytes());
for p in pieces {
out.extend(p.to_le_bytes());
}
out.extend((window.len() as u32).to_le_bytes());
}
}
}
out
}

/// [`compress_frame`] without the coding, for a page that codes frames on
/// other cores: the output is `u32 count` then, per frame the input buffer
/// completes, `u8 0, u64 frameLength, u64 archiveLength, archive` for a
/// finished repeat, or
/// `u8 1, u64 frameLength, u32 pieceCount, u64 piece*, u32 window` for a frame
/// to code with [`code_frame`] after the `window` frames before it. Frames
/// follow the stream's input back to back, repeats included.
#[no_mangle]
pub extern "C" fn plan_frame() -> i32 {
finish(INPUT.with(|input| {
Ok(plan_records(ENCODER.with(|encoder| {
encoder.borrow_mut().plan(&input.borrow())
})))
}))
}

/// [`compress_end`] without the coding (see [`plan_frame`]).
#[no_mangle]
pub extern "C" fn plan_end() -> i32 {
finish(Ok(plan_records(
ENCODER.with(|encoder| encoder.borrow_mut().plan_end()),
)))
}

/// Code one planned frame. The input buffer holds `u32 pieceCount, u64
/// piece*, u32 window, u64 windowLength*`, then the window frames and the
/// frame, back to back. With `which` 0 or 1 the output is that scheduled
/// strategy's archive, or empty when it cannot beat the raw archive (pass
/// both to [`choose_frame`]); with any other `which`, the frame's archive.
#[no_mangle]
pub extern "C" fn code_frame(which: i32, steps: u64) -> i32 {
finish(INPUT.with(|input| {
let input = input.borrow();
let mut f = Fields {
bytes: &input,
at: 0,
};
let pieces = f.pieces()?;
let lengths: Vec<usize> = (0..f.u32()?).map(|_| f.len()).collect::<Result<_, _>>()?;
let window: Vec<&[u8]> = lengths
.iter()
.map(|&n| f.take(n))
.collect::<Result<_, _>>()?;
let frame = &f.bytes[f.at..];
if pieces.iter().sum::<u64>() != frame.len() as u64 {
return Err(uorc::Error::Decode);
}
match which {
0 | 1 => Ok(
uorc::compress_strategy(frame, &pieces, &window, which as usize)
.unwrap_or_default(),
),
_ => uorc::Planned::Code {
bytes: frame.to_vec(),
pieces,
window: window.iter().map(|w| w.to_vec()).collect(),
}
.code(steps),
}
}))
}

/// The frame's archive from its two strategies' outputs of [`code_frame`]:
/// the input buffer holds `u32 pieceCount, u64 piece*, u64 frameLength,
/// frame, u64 firstLength, first, u64 secondLength, second` (an empty archive
/// for a strategy that could not beat raw).
#[no_mangle]
pub extern "C" fn choose_frame() -> i32 {
finish(INPUT.with(|input| {
let input = input.borrow();
let mut f = Fields {
bytes: &input,
at: 0,
};
let pieces = f.pieces()?;
let n = f.len()?;
let frame = f.take(n)?;
let mut archive = || -> Result<Option<Vec<u8>>, uorc::Error> {
let n = f.len()?;
Ok(Some(f.take(n)?.to_vec()).filter(|a| !a.is_empty()))
};
let (first, second) = (archive()?, archive()?);
f.end()?;
if pieces.iter().sum::<u64>() != frame.len() as u64 {
return Err(uorc::Error::Decode);
}
Ok(uorc::choose(frame, &pieces, first, second))
}))
}

#[cfg(target_arch = "wasm32")]
#[link(wasm_import_module = "uorc")]
extern "C" {
/// The page's progress callback: bytes done and length of the pass.
#[link_name = "progress"]
fn page_progress(done: f64, total: f64);
}

#[cfg(target_arch = "wasm32")]
fn report(done: u64, total: u64) {
// SAFETY: an imported function of two numbers with no other effect on
// this module's memory.
unsafe { page_progress(done as f64, total as f64) }
}

/// Report every coding and decoding pass to the page's `uorc.progress`
/// import (`on` nonzero), or stop reporting.
#[no_mangle]
pub extern "C" fn report_progress(on: i32) {
#[cfg(target_arch = "wasm32")]
uorc::ncm::set_progress((on != 0).then_some(report as uorc::ncm::Progress));
#[cfg(not(target_arch = "wasm32"))]
let _ = on;
}

/// Start a file tree streamed through the frame operations: the input
/// buffer lists its entries as `u32 count` then, per entry, `u8 kind`,
/// `u32 pathLength`, path, `u64 length` (zero for a directory). The output is
Expand Down Expand Up @@ -553,4 +734,101 @@ mod tests {
}
assert!(files.next().is_none());
}

/// Frames planned in one place and coded elsewhere, a strategy at a
/// time, write the library's stream: what the page's workers do.
#[test]
fn planned_frames_coded_by_strategy_are_the_library_stream() {
let noise = |n: usize, seed: u32| -> Vec<u8> {
let mut x = seed;
(0..n)
.map(|_| {
x = x.wrapping_mul(1_103_515_245).wrapping_add(12_345);
b"abcdefgh \n"[(x >> 24) as usize % 11]
})
.collect()
};
let a = noise(uorc::FRAME, 1);
// a frame, the same frame again (a reference frame), then a short tail
let input = [&a[..], &a[..], &noise(70_000, 3)].concat();
window_reset();
let mut records = Vec::new();
for piece in input.chunks(5_000_000) {
let (code, out) = run(piece, || plan_frame());
assert_eq!(code, OK);
records.push(out);
}
records.push(run(&[], || plan_end()).1);
let (mut stream, mut at, mut frames) = (Vec::new(), 0usize, Vec::<&[u8]>::new());
let mut references = 0;
for record in &records {
let mut f = Fields {
bytes: record,
at: 0,
};
for _ in 0..f.u32().unwrap() {
let kind = f.take(1).unwrap()[0];
let n = f.len().unwrap();
let frame = &input[at..at + n];
if kind == 0 {
let len = f.len().unwrap();
stream.extend(f.take(len).unwrap());
references += 1;
} else {
let pieces = f.pieces().unwrap();
let before = &frames[frames.len() - f.u32().unwrap()..];
let mut head = (pieces.len() as u32).to_le_bytes().to_vec();
for p in &pieces {
head.extend(p.to_le_bytes());
}
let mut request = head.clone();
request.extend((before.len() as u32).to_le_bytes());
for w in before {
request.extend((w.len() as u64).to_le_bytes());
}
for w in before {
request.extend(*w);
}
request.extend(frame);
let (c0, first) = run(&request, || code_frame(0, 0));
let (c1, second) = run(&request, || code_frame(1, 0));
assert_eq!((c0, c1), (OK, OK));
let mut choice = head;
for part in [frame, &first[..], &second[..]] {
choice.extend((part.len() as u64).to_le_bytes());
choice.extend(part);
}
let (code, archive) = run(&choice, || choose_frame());
assert_eq!(code, OK);
assert_eq!(archive, run(&request, || code_frame(-1, 0)).1);
stream.extend(archive);
}
frames.push(frame);
at += n;
}
}
assert!(references > 0);
assert_eq!(at, input.len());
assert_eq!(stream, uorc::compress(&input, 0).expect("library stream"));
let archives = uorc::frames(&stream).expect("frames");
assert!(archives.iter().any(|a| {
uorc::inspect(a).expect("inspect").representation == uorc::Representation::Reference
}));
window_reset();
for archive in &archives {
if run(archive, || may_be_reference()).0 == 1 {
run(archive, || {
note_frame();
OK
});
}
}
let mut back = Vec::new();
for archive in &archives {
let (code, output) = run(archive, || decompress_frame());
assert_eq!(code, OK);
back.extend(output);
}
assert!(back == input);
}
}
8 changes: 6 additions & 2 deletions crates/uorc/src/archive.rs
Original file line number Diff line number Diff line change
Expand Up @@ -867,11 +867,15 @@ pub fn window_decode(data: &[u8], window: &[&[u8]]) -> Option<Vec<u8>> {
return None;
}
let (codec, era, payload) = window_payload(&pre[start..], k)?;
let history = window.concat();
let decode = || {
if !codec.x86() {
return decode_with(codec, &history, payload, n);
return match codec.ncm_profile() {
// The model reads the window's frames in place.
Some(p) => crate::ncm::decode_canonical_window(window, payload, n, p),
None => crate::ulz::decode_with(&window.concat(), payload, n),
};
}
let history = window.concat();
let h = crate::x86::filter(&history, false, 0).0;
let out = decode_with(codec, &h, payload, n)?;
Some(crate::x86::filter(&out, true, history.len() as u64).0)
Expand Down
Loading
Loading