From 8fedf9551eceab41bb585c968c47c994ad27818e Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 9 Nov 2026 15:38:58 +0000 Subject: [PATCH] client: inspect, verify, safe output writes, multi-file archives - uorc inspect: frame header (scope, representation, object lengths, sizes) and the UORC-SHA256-CHAIN-v1 seal, without decoding; walks archives written back to back. --json for machine output. - uorc verify: decode in memory, check seal and declared lengths, and optionally compare with an original. Distinct exit codes: 2 usage, 3 I/O, 4 invalid archive, 5 budget, 6 overflow, 7 differs from the original. - Outputs go to a temporary file beside the destination and are moved into place with a no-replace hard link (or rename under --force); a failed run leaves nothing behind. - Ordered objects: compress_objects/decompress_objects code the concatenation once and record each length in the frame's objectLengths (resealed); decompress relabels to one object for the certified path. - Trees: a directory or several paths become [manifest, file...]; restore refuses absolute and '..' paths and symlinked parents; --resume keeps files already restored. - wasm exports inspect, verify, compress_tree, decompress_tree; the Pages site gains multi-file/folder compress, per-file download, Inspect, Verify. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013Asa3LpDk5B19xbzm4jSei --- Cargo.lock | 1 + crates/uorc-wasm/src/lib.rs | 132 ++++++++- crates/uorc/Cargo.toml | 1 + crates/uorc/src/bin/uorc.rs | 548 ++++++++++++++++++++++++++++++++---- crates/uorc/src/frame.rs | 347 +++++++++++++++++++++++ crates/uorc/src/lib.rs | 163 ++++++++++- crates/uorc/src/tree.rs | 212 ++++++++++++++ crates/uorc/tests/cli.rs | 216 ++++++++++++++ web/app.js | 110 ++++++-- web/index.html | 43 ++- web/uorc.js | 58 +++- 11 files changed, 1744 insertions(+), 87 deletions(-) create mode 100644 crates/uorc/src/frame.rs create mode 100644 crates/uorc/src/tree.rs create mode 100644 crates/uorc/tests/cli.rs diff --git a/Cargo.lock b/Cargo.lock index 80233c4d..fcb707f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -659,6 +659,7 @@ dependencies = [ name = "uorc" version = "0.1.0" dependencies = [ + "sha2 0.11.1", "uorc_compress", "uorc_decompress", ] diff --git a/crates/uorc-wasm/src/lib.rs b/crates/uorc-wasm/src/lib.rs index c0675943..015489a3 100644 --- a/crates/uorc-wasm/src/lib.rs +++ b/crates/uorc-wasm/src/lib.rs @@ -9,6 +9,7 @@ use std::cell::RefCell; thread_local! { static INPUT: RefCell> = const { RefCell::new(Vec::new()) }; static OUTPUT: RefCell> = const { RefCell::new(Vec::new()) }; + static ORIGINAL: RefCell> = const { RefCell::new(Vec::new()) }; } /// Operation succeeded; the result is in the output buffer. @@ -19,16 +20,27 @@ pub const BUDGET: i32 = 1; pub const OVERFLOW: i32 = 2; /// See [`uorc::Error::Decode`]. pub const DECODE: i32 = 3; +/// See [`uorc::Error::Mismatch`]. +pub const MISMATCH: i32 = 4; + +fn resize(buffer: &RefCell>, len: usize) -> *mut u8 { + let mut buffer = buffer.borrow_mut(); + buffer.clear(); + buffer.resize(len, 0); + buffer.as_mut_ptr() +} /// Resize the input buffer to `len` bytes and return its address in memory. #[no_mangle] pub extern "C" fn input_buffer(len: usize) -> *mut u8 { - INPUT.with(|input| { - let mut input = input.borrow_mut(); - input.clear(); - input.resize(len, 0); - input.as_mut_ptr() - }) + INPUT.with(|input| resize(input, len)) +} + +/// Resize the original buffer (the bytes [`verify`] compares against) to +/// `len` bytes and return its address in memory. +#[no_mangle] +pub extern "C" fn original_buffer(len: usize) -> *mut u8 { + ORIGINAL.with(|original| resize(original, len)) } /// Address of the last operation's output. @@ -49,6 +61,7 @@ fn finish(result: Result, uorc::Error>) -> i32 { Err(uorc::Error::Budget) => (BUDGET, Vec::new()), Err(uorc::Error::Overflow) => (OVERFLOW, Vec::new()), Err(uorc::Error::Decode) => (DECODE, Vec::new()), + Err(uorc::Error::Mismatch) => (MISMATCH, Vec::new()), }; OUTPUT.with(|output| *output.borrow_mut() = bytes); code @@ -65,3 +78,110 @@ pub extern "C" fn compress(steps: u64) -> i32 { pub extern "C" fn decompress() -> i32 { finish(INPUT.with(|input| uorc::decompress(&input.borrow(), uorc::DEFAULT_DECODE_WORK))) } + +/// Read the input archive's frame header and integrity seal without decoding; +/// the output is the description as JSON. +#[no_mangle] +pub extern "C" fn inspect() -> i32 { + finish( + INPUT.with(|input| uorc::inspect(&input.borrow()).map(|info| info.to_json().into_bytes())), + ) +} + +/// Decode the input (one archive or several back to back) in memory and check +/// it; with `compare` nonzero, also check that it reproduces the original +/// buffer. The output is a JSON array describing each archive. +#[no_mangle] +pub extern "C" fn verify(compare: i32) -> i32 { + finish(INPUT.with(|input| { + ORIGINAL.with(|original| { + let original = original.borrow(); + uorc::verify( + &input.borrow(), + (compare != 0).then_some(original.as_slice()), + uorc::DEFAULT_DECODE_WORK, + ) + .map(|infos| { + let all: Vec = infos.iter().map(uorc::Info::to_json).collect(); + format!("[{}]", all.join(",")).into_bytes() + }) + }) + })) +} + +/// Entries cross the wasm boundary as `u32 count` then, per entry, +/// `u8 kind` (0 file, 1 executable, 2 directory), `u32 pathLength`, path, +/// `u32 dataLength`, data; integers little-endian. +fn read_entries(bytes: &[u8]) -> Result, uorc::Error> { + use uorc::tree::{Entry, Kind}; + struct Cursor<'a>(&'a [u8]); + impl<'a> Cursor<'a> { + fn take(&mut self, n: usize) -> Result<&'a [u8], uorc::Error> { + if n > self.0.len() { + return Err(uorc::Error::Decode); + } + let (head, tail) = self.0.split_at(n); + self.0 = tail; + Ok(head) + } + fn word(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4 bytes")) as usize) + } + } + let mut c = Cursor(bytes); + let count = c.word()?; + let mut entries = Vec::new(); + for _ in 0..count { + let kind = match c.take(1)?[0] { + 0 => Kind::File, + 1 => Kind::Executable, + 2 => Kind::Directory, + _ => return Err(uorc::Error::Decode), + }; + let len = c.word()?; + let path = String::from_utf8(c.take(len)?.to_vec()).map_err(|_| uorc::Error::Decode)?; + let len = c.word()?; + let data = c.take(len)?.to_vec(); + entries.push(Entry { path, kind, data }); + } + if !c.0.is_empty() { + return Err(uorc::Error::Decode); + } + Ok(entries) +} + +fn write_entries(entries: &[uorc::tree::Entry]) -> Vec { + use uorc::tree::Kind; + let mut out = (entries.len() as u32).to_le_bytes().to_vec(); + for entry in entries { + out.push(match entry.kind { + Kind::File => 0, + Kind::Executable => 1, + Kind::Directory => 2, + }); + out.extend((entry.path.len() as u32).to_le_bytes()); + out.extend(entry.path.as_bytes()); + out.extend((entry.data.len() as u32).to_le_bytes()); + out.extend(&entry.data); + } + out +} + +/// Compress the entries in the input buffer (see [`read_entries`]) into one +/// archive of ordered objects. +#[no_mangle] +pub extern "C" fn compress_tree(steps: u64) -> i32 { + finish(INPUT.with(|input| { + read_entries(&input.borrow()).and_then(|entries| uorc::compress_tree(&entries, steps)) + })) +} + +/// Restore the entries a tree archive in the input buffer stores; the output +/// is in the [`read_entries`] layout. A one-object archive is not a tree. +#[no_mangle] +pub extern "C" fn decompress_tree() -> i32 { + finish(INPUT.with(|input| { + uorc::decompress_tree(&input.borrow(), uorc::DEFAULT_DECODE_WORK) + .map(|entries| write_entries(&entries)) + })) +} diff --git a/crates/uorc/Cargo.toml b/crates/uorc/Cargo.toml index b2e5aa1e..bb4db8aa 100644 --- a/crates/uorc/Cargo.toml +++ b/crates/uorc/Cargo.toml @@ -18,6 +18,7 @@ name = "uorc" path = "src/bin/uorc.rs" [dependencies] +sha2 = "0.11.1" uorc_compress = { version = "0.1.0", path = "../../generated/rust/uorc_compress" } uorc_decompress = { version = "0.1.0", path = "../../generated/rust/uorc_decompress" } diff --git a/crates/uorc/src/bin/uorc.rs b/crates/uorc/src/bin/uorc.rs index 5f688bbe..96376e4b 100644 --- a/crates/uorc/src/bin/uorc.rs +++ b/crates/uorc/src/bin/uorc.rs @@ -1,98 +1,437 @@ //! `uorc`: the command-line client. -use std::io::{Read, Write}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; use std::process::ExitCode; use std::time::Instant; const USAGE: &str = "usage: - uorc compress [--steps N] [--stats] [-o ] - uorc decompress [--work N] [--stats] [-o ] + uorc compress [--steps N] [--force] [--stats] [-o ] + uorc compress [--steps N] [--force] [--stats] ... -o + uorc decompress [--work N] [--force|--resume] [--stats] [-o ] + uorc inspect [--json] + uorc verify [--work N] [--json] [original] uorc --version Compress writes the smaller of the ulz kernel archive and the raw archive; --steps N also runs the program search for N steps and keeps the smallest. Without -o, compress writes .uorc and decompress strips a .uorc suffix -(or writes .out). '-' reads stdin or writes stdout."; +(or writes .out). '-' reads stdin or writes stdout. +Given a directory or several paths, compress stores them as one archive of +ordered objects: a manifest of relative paths, then every file in order +(symbolic links are skipped). Decompressing such an archive restores the tree +under the output directory (default: the archive name without .uorc). Paths +that are absolute or contain '..' are refused. --resume keeps files that +already exist with the right contents, so an interrupted restore can finish. + +Outputs are written to a temporary file next to the destination and renamed +into place, so a failed run leaves nothing behind. An existing output is never +replaced unless --force is given. + +inspect reads the frame header and checks the integrity seal without decoding. +verify decodes in memory and checks integrity and declared lengths, and, given +an original, that the archive reproduces it byte for byte. + +exit status: 0 ok, 2 usage, 3 I/O (including a refused overwrite), +4 not a valid archive or failed integrity, 5 step budget, 6 overflow, +7 archive does not match the original."; + +/// A failure and the exit status it maps to. +struct Failure { + code: u8, + message: String, +} + +const USAGE_ERROR: u8 = 2; +const IO_ERROR: u8 = 3; + +fn usage(message: impl Into) -> Failure { + Failure { + code: USAGE_ERROR, + message: format!("{}\n\n{USAGE}", message.into()), + } +} + +fn io_failure(path: &str, error: io::Error) -> Failure { + Failure { + code: IO_ERROR, + message: format!("{path}: {error}"), + } +} + +impl From for Failure { + fn from(error: uorc::Error) -> Self { + let code = match error { + uorc::Error::Decode => 4, + uorc::Error::Budget => 5, + uorc::Error::Overflow => 6, + uorc::Error::Mismatch => 7, + }; + Failure { + code, + message: error.to_string(), + } + } +} + +#[derive(Default)] struct Options { command: String, budget: Option, stats: bool, - input: String, + force: bool, + resume: bool, + json: bool, + paths: Vec, output: Option, } -fn parse(mut args: impl Iterator) -> Result { - let command = args.next().ok_or("missing command")?; - let (mut budget, mut stats, mut input, mut output) = (None, false, None, None); +fn parse(mut args: impl Iterator) -> Result { + let mut options = Options { + command: args.next().ok_or_else(|| usage("missing command"))?, + ..Options::default() + }; while let Some(arg) = args.next() { match arg.as_str() { "--steps" | "--work" => { - let value = args.next().ok_or_else(|| format!("{arg} needs a number"))?; - budget = Some( + let value = args + .next() + .ok_or_else(|| usage(format!("{arg} needs a number")))?; + options.budget = Some( value .parse::() - .map_err(|_| format!("{arg}: not a number: {value}"))?, + .map_err(|_| usage(format!("{arg}: not a number: {value}")))?, ); } - "--stats" => stats = true, - "-o" => output = Some(args.next().ok_or("-o needs a path")?), - _ if input.is_none() => input = Some(arg), - _ => return Err(format!("unexpected argument: {arg}")), + "--stats" => options.stats = true, + "--force" | "-f" => options.force = true, + "--json" => options.json = true, + "--resume" => options.resume = true, + "-o" => options.output = Some(args.next().ok_or_else(|| usage("-o needs a path"))?), + _ => options.paths.push(arg), } } - Ok(Options { - command, - budget, - stats, - input: input.ok_or("missing input")?, - output, - }) + Ok(options) } -fn read(path: &str) -> std::io::Result> { +fn read(path: &str) -> Result, Failure> { if path == "-" { let mut bytes = Vec::new(); - std::io::stdin().read_to_end(&mut bytes)?; + io::stdin() + .read_to_end(&mut bytes) + .map_err(|e| io_failure(path, e))?; Ok(bytes) } else { - std::fs::read(path) + fs::read(path).map_err(|e| io_failure(path, e)) + } +} + +/// A file being written next to its destination. It is removed on drop +/// unless [`Pending::commit`] moved it into place. +struct Pending { + temp: PathBuf, + file: Option, +} + +impl Pending { + fn create(dest: &Path) -> io::Result { + let dir = match dest.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir, + _ => Path::new("."), + }; + let name = dest + .file_name() + .map_or_else(|| "output".into(), |n| n.to_string_lossy().into_owned()); + for attempt in 0u32.. { + let temp = dir.join(format!(".{name}.{}.{attempt}.uorc-tmp", std::process::id())); + match OpenOptions::new().write(true).create_new(true).open(&temp) { + Ok(file) => { + return Ok(Pending { + temp, + file: Some(file), + }) + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists && attempt < 1000 => {} + Err(e) => return Err(e), + } + } + unreachable!("the attempt loop returns") + } + + fn write_all(&mut self, bytes: &[u8]) -> io::Result<()> { + self.file.as_mut().expect("open").write_all(bytes) } + + /// Flush to disk and move into place: atomically, and without replacing + /// an existing `dest` unless `force`. + fn commit(mut self, dest: &Path, force: bool) -> io::Result<()> { + let file = self.file.take().expect("open"); + file.sync_all()?; + drop(file); + if force { + fs::rename(&self.temp, dest)?; + } else { + // A hard link fails if `dest` exists, so no other writer can be + // overwritten between a check and the move. Filesystems without + // hard links fall back to a check then rename. + match fs::hard_link(&self.temp, dest) { + Ok(()) => fs::remove_file(&self.temp)?, + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => return Err(exists(dest)), + Err(_) if dest.exists() => return Err(exists(dest)), + Err(_) => fs::rename(&self.temp, dest)?, + } + } + self.temp = PathBuf::new(); + Ok(()) + } +} + +impl Drop for Pending { + fn drop(&mut self) { + drop(self.file.take()); + if !self.temp.as_os_str().is_empty() { + let _ = fs::remove_file(&self.temp); + } + } +} + +fn exists(dest: &Path) -> io::Error { + io::Error::new( + io::ErrorKind::AlreadyExists, + format!("{} exists (use --force to replace it)", dest.display()), + ) +} + +/// Refuse early, before any work, when the output already exists. +fn check_free(path: &str, force: bool) -> Result<(), Failure> { + if path != "-" && !force && Path::new(path).exists() { + return Err(io_failure(path, exists(Path::new(path)))); + } + Ok(()) } -fn write(path: &str, bytes: &[u8]) -> std::io::Result<()> { +fn write(path: &str, bytes: &[u8], force: bool) -> Result<(), Failure> { + let fail = |e| io_failure(path, e); if path == "-" { - std::io::stdout().write_all(bytes) + let mut out = io::stdout().lock(); + return out + .write_all(bytes) + .and_then(|()| out.flush()) + .map_err(fail); + } + let dest = Path::new(path); + let mut pending = Pending::create(dest).map_err(fail)?; + pending.write_all(bytes).map_err(fail)?; + pending.commit(dest, force).map_err(fail) +} + +fn one_path(options: &Options) -> Result<&str, Failure> { + match options.paths.as_slice() { + [path] => Ok(path), + [] => Err(usage("missing input")), + [_, extra, ..] => Err(usage(format!("unexpected argument: {extra}"))), + } +} + +fn is_tree_request(options: &Options) -> bool { + options.paths.len() > 1 || (options.paths.len() == 1 && Path::new(&options.paths[0]).is_dir()) +} + +/// Collect `root` (a file or directory) into `entries`, with paths relative +/// to the directory that holds `root`. +fn collect(root: &Path, name: &str, entries: &mut Vec) -> Result<(), Failure> { + use uorc::tree::{Entry, Kind}; + let shown = root.display().to_string(); + let meta = fs::symlink_metadata(root).map_err(|e| io_failure(&shown, e))?; + if meta.file_type().is_symlink() { + eprintln!("uorc: skipping symbolic link {shown}"); + } else if meta.is_dir() { + entries.push(Entry { + path: name.to_owned(), + kind: Kind::Directory, + data: Vec::new(), + }); + let mut children: Vec<_> = fs::read_dir(root) + .map_err(|e| io_failure(&shown, e))? + .collect::>() + .map_err(|e| io_failure(&shown, e))?; + children.sort_by_key(fs::DirEntry::file_name); + for child in children { + let child_name = child.file_name(); + let child_name = child_name.to_str().ok_or_else(|| Failure { + code: IO_ERROR, + message: format!("{}: file name is not UTF-8", child.path().display()), + })?; + collect(&child.path(), &format!("{name}/{child_name}"), entries)?; + } + } else if meta.is_file() { + entries.push(Entry { + path: name.to_owned(), + kind: if executable(&meta) { + Kind::Executable + } else { + Kind::File + }, + data: fs::read(root).map_err(|e| io_failure(&shown, e))?, + }); } else { - std::fs::write(path, bytes) + eprintln!("uorc: skipping special file {shown}"); } + Ok(()) } -fn run(options: Options) -> Result<(), String> { - let input = read(&options.input).map_err(|e| format!("{}: {e}", options.input))?; - let started = Instant::now(); - let (output, default_path) = match options.command.as_str() { - "compress" => ( - uorc::compress(&input, options.budget.unwrap_or(uorc::DEFAULT_STEPS)), - format!("{}.uorc", options.input), - ), - "decompress" => ( - uorc::decompress(&input, options.budget.unwrap_or(uorc::DEFAULT_DECODE_WORK)), - options - .input - .strip_suffix(".uorc") - .map_or_else(|| format!("{}.out", options.input), str::to_owned), - ), - other => return Err(format!("unknown command: {other}")), +#[cfg(unix)] +fn executable(meta: &fs::Metadata) -> bool { + use std::os::unix::fs::PermissionsExt; + meta.permissions().mode() & 0o111 != 0 +} + +#[cfg(not(unix))] +fn executable(_: &fs::Metadata) -> bool { + false +} + +#[cfg(unix)] +fn set_executable(path: &Path) -> io::Result<()> { + use std::os::unix::fs::PermissionsExt; + let mut permissions = fs::metadata(path)?.permissions(); + permissions.set_mode(permissions.mode() | 0o111); + fs::set_permissions(path, permissions) +} + +#[cfg(not(unix))] +fn set_executable(_: &Path) -> io::Result<()> { + Ok(()) +} + +fn compress_tree(options: &Options) -> Result<(), Failure> { + let output = match (&options.output, options.paths.as_slice()) { + (Some(path), _) => path.clone(), + (None, [dir]) => format!("{}.uorc", dir.trim_end_matches('/')), + (None, _) => return Err(usage("several inputs need -o ")), }; - let output = output.map_err(|e| e.to_string())?; + check_free(&output, options.force)?; + let mut entries = Vec::new(); + for path in &options.paths { + let root = Path::new(path); + let name = root + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| usage(format!("{path}: no usable file name")))?; + collect(root, name, &mut entries)?; + } + let mut seen = std::collections::HashSet::new(); + if let Some(dup) = entries.iter().find(|e| !seen.insert(e.path.clone())) { + return Err(usage(format!("two inputs are both named {}", dup.path))); + } + let input_bytes: usize = entries.iter().map(|e| e.data.len()).sum(); + let started = Instant::now(); + let archive = uorc::compress_tree(&entries, options.budget.unwrap_or(uorc::DEFAULT_STEPS))?; let elapsed = started.elapsed(); - let path = match options.output { - Some(path) => path, - None if options.input == "-" => "-".to_owned(), - None => default_path, + write(&output, &archive, options.force)?; + if options.stats { + eprintln!( + "{{\"command\":\"compress\",\"entries\":{},\"input_bytes\":{input_bytes},\"output_bytes\":{},\"seconds\":{:.6}}}", + entries.len(), + archive.len(), + elapsed.as_secs_f64() + ); + } + Ok(()) +} + +/// Restore a tree under `dest`. With `resume`, files already holding the +/// right bytes are kept; otherwise an existing file is refused unless `force`. +fn restore( + entries: &[uorc::tree::Entry], + dest: &Path, + options: &Options, +) -> Result { + use uorc::tree::Kind; + let shown = dest.display().to_string(); + fs::create_dir_all(dest).map_err(|e| io_failure(&shown, e))?; + let mut written = 0; + for entry in entries { + let path = dest.join(&entry.path); + let shown = path.display().to_string(); + // A path component that is a symbolic link could lead outside `dest`. + if let Some(parent) = Path::new(&entry.path).parent() { + let mut at = dest.to_path_buf(); + for part in parent.components() { + at.push(part); + if fs::symlink_metadata(&at).is_ok_and(|m| m.file_type().is_symlink()) { + return Err(io_failure( + &shown, + io::Error::other("a parent directory is a symbolic link"), + )); + } + } + } + if entry.kind == Kind::Directory { + fs::create_dir_all(&path).map_err(|e| io_failure(&shown, e))?; + continue; + } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).map_err(|e| io_failure(&shown, e))?; + } + if options.resume && fs::read(&path).is_ok_and(|bytes| bytes == entry.data) { + continue; + } + write(&shown, &entry.data, options.force || options.resume)?; + if entry.kind == Kind::Executable { + set_executable(&path).map_err(|e| io_failure(&shown, e))?; + } + written += 1; + } + Ok(written) +} + +fn transform(options: &Options) -> Result<(), Failure> { + let compress = options.command == "compress"; + if compress && is_tree_request(options) { + return compress_tree(options); + } + let input_path = one_path(options)?; + let path = match &options.output { + Some(path) => path.clone(), + None if input_path == "-" => "-".to_owned(), + None if compress => format!("{input_path}.uorc"), + None => input_path + .strip_suffix(".uorc") + .map_or_else(|| format!("{input_path}.out"), str::to_owned), + }; + let input = read(input_path)?; + if !compress && uorc::inspect(&input).is_ok_and(|info| info.object_lengths.len() != 1) { + if path == "-" { + return Err(usage( + "a multi-object archive restores to a directory; give -o ", + )); + } + let started = Instant::now(); + let entries = + uorc::decompress_tree(&input, options.budget.unwrap_or(uorc::DEFAULT_DECODE_WORK))?; + let written = restore(&entries, Path::new(&path), options)?; + if options.stats { + eprintln!( + "{{\"command\":\"decompress\",\"entries\":{},\"written\":{written},\"input_bytes\":{},\"seconds\":{:.6}}}", + entries.len(), + input.len(), + started.elapsed().as_secs_f64() + ); + } + return Ok(()); + } + check_free(&path, options.force)?; + let started = Instant::now(); + let output = if compress { + uorc::compress(&input, options.budget.unwrap_or(uorc::DEFAULT_STEPS))? + } else { + uorc::decompress(&input, options.budget.unwrap_or(uorc::DEFAULT_DECODE_WORK))? }; - write(&path, &output).map_err(|e| format!("{path}: {e}"))?; + let elapsed = started.elapsed(); + write(&path, &output, options.force)?; if options.stats { eprintln!( "{{\"command\":\"{}\",\"input_bytes\":{},\"output_bytes\":{},\"seconds\":{:.6}}}", @@ -105,6 +444,101 @@ fn run(options: Options) -> Result<(), String> { Ok(()) } +fn describe(info: &uorc::Info) -> String { + let scope = info.basis.map_or_else( + || "self-contained".to_owned(), + |b| { + let hex: String = b.iter().map(|x| format!("{x:02x}")).collect(); + format!("conditional on {hex}") + }, + ); + format!( + "format {} (machine profile {})\n\ + scope {scope}\n\ + representation {}\n\ + objects {} ({} bytes)\n\ + body {} bytes\n\ + archive {} bytes\n\ + integrity {} {} ({})", + info.format, + info.profile, + info.representation.name(), + info.object_lengths.len(), + info.output_len, + info.body_len, + info.archive_len, + info.integrity_profile(), + info.digest_hex(), + if info.integrity_ok { "ok" } else { "MISMATCH" }, + ) +} + +fn inspect(options: &Options) -> Result<(), Failure> { + let stream = read(one_path(options)?)?; + let infos = uorc::frames(&stream)? + .into_iter() + .map(uorc::inspect) + .collect::, _>>()?; + if infos.is_empty() { + return Err(uorc::Error::Decode.into()); + } + if options.json { + let all: Vec = infos.iter().map(uorc::Info::to_json).collect(); + println!("[{}]", all.join(",")); + } else { + let all: Vec = infos.iter().map(describe).collect(); + println!("{}", all.join("\n\n")); + } + if infos.iter().all(|info| info.integrity_ok) { + Ok(()) + } else { + Err(Failure { + code: 4, + message: "integrity digest does not match the archive".to_owned(), + }) + } +} + +fn verify(options: &Options) -> Result<(), Failure> { + let (archive_path, original_path) = match options.paths.as_slice() { + [archive] => (archive, None), + [archive, original] => (archive, Some(original)), + [] => return Err(usage("missing archive")), + [_, _, extra, ..] => return Err(usage(format!("unexpected argument: {extra}"))), + }; + let archive = read(archive_path)?; + let original = original_path.map(|p| read(p)).transpose()?; + let started = Instant::now(); + let infos = uorc::verify( + &archive, + original.as_deref(), + options.budget.unwrap_or(uorc::DEFAULT_DECODE_WORK), + )?; + if options.json { + let all: Vec = infos.iter().map(uorc::Info::to_json).collect(); + println!( + "{{\"ok\":true,\"compared\":{},\"seconds\":{:.6},\"archives\":[{}]}}", + original.is_some(), + started.elapsed().as_secs_f64(), + all.join(",") + ); + } else { + println!( + "ok: {} archive(s), {} objects, {} bytes from {} archive bytes{}", + infos.len(), + infos.iter().map(|i| i.object_lengths.len()).sum::(), + infos.iter().map(|i| i.output_len).sum::(), + archive.len(), + if original.is_some() { + ", identical to the original" + } else { + "" + } + ); + } + Ok(()) +} + fn main() -> ExitCode { let mut args = std::env::args().skip(1).peekable(); match args.peek().map(String::as_str) { @@ -118,11 +552,17 @@ fn main() -> ExitCode { } _ => {} } - match parse(args).and_then(run) { + let result = parse(args).and_then(|options| match options.command.as_str() { + "compress" | "decompress" => transform(&options), + "inspect" => inspect(&options), + "verify" => verify(&options), + other => Err(usage(format!("unknown command: {other}"))), + }); + match result { Ok(()) => ExitCode::SUCCESS, - Err(message) => { - eprintln!("uorc: {message}\n\n{USAGE}"); - ExitCode::FAILURE + Err(failure) => { + eprintln!("uorc: {}", failure.message); + ExitCode::from(failure.code) } } } diff --git a/crates/uorc/src/frame.rs b/crates/uorc/src/frame.rs new file mode 100644 index 00000000..78b8468e --- /dev/null +++ b/crates/uorc/src/frame.rs @@ -0,0 +1,347 @@ +//! Reading a UORC archive's frame header and integrity seal without decoding +//! its body. +//! +//! The layout is the one `Uorc.Wire.FrameParse` accepts: +//! +//! ```text +//! "UORC" format profile scope representation +//! uleb(objectCount) uleb(objectLength)* uleb(bodyLength) +//! basis[32] -- only when scope = 1 (conditional) +//! body[bodyLength] +//! digest[32] -- UORC-SHA256-CHAIN-v1 over everything before it +//! ``` +//! +//! Numbers are shortest-form ULEB128. A raw body's length equals the sum of +//! the object lengths. This module only reads that framing; the body is +//! decoded and checked by the certified decoder in [`crate::decompress`]. + +use sha2::{Digest, Sha256}; + +use crate::Error; + +const MAGIC: &[u8; 4] = b"UORC"; +const DIGEST_LEN: usize = 32; +const BASIS_LEN: usize = 32; +const CHAIN_TAG: &[u8] = b"UORC-SHA256-CHAIN-v1\0"; + +/// How an archive's body produces its output. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Representation { + /// The body is the output bytes, stored as is. + Raw, + /// The body is the ulz decoder kernel graph with an LZ + range-coded payload. + UlzKernel, + /// The body is any other UORC program graph (for example a search result). + Program, +} + +impl Representation { + /// A short lowercase name: `raw`, `ulz-kernel` or `program`. + pub fn name(self) -> &'static str { + match self { + Representation::Raw => "raw", + Representation::UlzKernel => "ulz-kernel", + Representation::Program => "program", + } + } +} + +/// What an archive's frame header says, read without decoding the body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Info { + /// Archive format version (2 for this client). + pub format: u8, + /// Machine profile (1 for this client). + pub profile: u8, + /// The basis identity of a conditional archive; `None` when self-contained. + pub basis: Option<[u8; BASIS_LEN]>, + /// How the body produces the output. + pub representation: Representation, + /// Byte length of each output object, in order. + pub object_lengths: Vec, + /// Total output length: the sum of `object_lengths`. + pub output_len: u64, + /// Length of the body. + pub body_len: u64, + /// Length of the whole archive. + pub archive_len: u64, + /// The sealed integrity digest at the end of the archive. + pub digest: [u8; DIGEST_LEN], + /// Whether the digest matches the archive's bytes. + pub integrity_ok: bool, +} + +impl Info { + /// Name of the integrity profile that sealed the archive. + pub fn integrity_profile(&self) -> &'static str { + "UORC-SHA256-CHAIN-v1" + } + + /// The digest as lowercase hex. + pub fn digest_hex(&self) -> String { + hex(&self.digest) + } + + /// One JSON object describing the archive (no external JSON dependency). + pub fn to_json(&self) -> String { + let lengths: Vec = self.object_lengths.iter().map(u64::to_string).collect(); + format!( + "{{\"format\":{},\"profile\":{},\"scope\":\"{}\",\"basis\":{},\"representation\":\"{}\",\ + \"objects\":{},\"object_lengths\":[{}],\"output_bytes\":{},\"body_bytes\":{},\ + \"archive_bytes\":{},\"integrity\":\"{}\",\"digest\":\"{}\",\"integrity_ok\":{}}}", + self.format, + self.profile, + if self.basis.is_some() { "conditional" } else { "self-contained" }, + self.basis.map_or_else(|| "null".to_owned(), |b| format!("\"{}\"", hex(&b))), + self.representation.name(), + self.object_lengths.len(), + lengths.join(","), + self.output_len, + self.body_len, + self.archive_len, + self.integrity_profile(), + self.digest_hex(), + self.integrity_ok, + ) + } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +struct Reader<'a> { + data: &'a [u8], + at: usize, +} + +impl<'a> Reader<'a> { + fn byte(&mut self) -> Result { + let b = *self.data.get(self.at).ok_or(Error::Decode)?; + self.at += 1; + Ok(b) + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], Error> { + let end = self.at.checked_add(n).ok_or(Error::Decode)?; + let s = self.data.get(self.at..end).ok_or(Error::Decode)?; + self.at = end; + Ok(s) + } + + /// Shortest-form ULEB128 that fits in 64 bits. + fn uleb(&mut self) -> Result { + let (mut value, mut shift) = (0u64, 0u32); + loop { + let b = self.byte()?; + let part = u64::from(b & 0x7f); + if shift >= 64 || (shift == 63 && part > 1) { + return Err(Error::Decode); + } + value |= part << shift; + if b & 0x80 == 0 { + if b == 0 && shift > 0 { + return Err(Error::Decode); // non-shortest + } + return Ok(value); + } + shift += 7; + } + } +} + +/// The ulz kernel graph's bytes before its payload length, taken from the +/// codec source so this check follows any regeneration of the kernel. +fn kernel_prefix() -> &'static [u8] { + use std::sync::OnceLock; + static PREFIX: OnceLock> = OnceLock::new(); + PREFIX.get_or_init(|| { + let source = include_str!("../../../client/src/Uorc/Codec/Archive.lex.tex"); + let def = source + .split("name: kernelPrefix,") + .nth(1) + .expect("kernelPrefix in Archive.lex.tex"); + let text = def + .split("body: 0x") + .nth(1) + .expect("kernelPrefix body") + .split(|c: char| !c.is_ascii_hexdigit()) + .next() + .expect("hex literal"); + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&text[i..i + 2], 16).expect("hex")) + .collect() + }) +} + +/// The UORC-SHA256-CHAIN-v1 digest of `preimage`. +pub(crate) fn chain_digest(preimage: &[u8]) -> [u8; DIGEST_LEN] { + let mut c: [u8; DIGEST_LEN] = Sha256::new() + .chain_update(CHAIN_TAG) + .chain_update([0u8]) + .finalize() + .into(); + let mut blocks = preimage.chunks_exact(64); + for block in &mut blocks { + c = Sha256::new() + .chain_update(CHAIN_TAG) + .chain_update([1u8]) + .chain_update(c) + .chain_update(block) + .finalize() + .into(); + } + let tail = blocks.remainder(); + Sha256::new() + .chain_update(CHAIN_TAG) + .chain_update([2u8]) + .chain_update(c) + .chain_update([tail.len() as u8]) + .chain_update(tail) + .finalize() + .into() +} + +/// Where an archive's fields sit, from [`parse`]. +pub(crate) struct Layout { + /// Offset of the body-length field (just after the object lengths). + body_len_at: usize, + /// End of the sealed preimage (start of the digest). + sealed: usize, + /// End of the archive. + end: usize, +} + +/// Parse one archive at the start of `data`, which may continue past it. +fn parse(data: &[u8]) -> Result<(Info, Layout), Error> { + let mut r = Reader { data, at: 0 }; + if r.take(4)? != MAGIC { + return Err(Error::Decode); + } + let format = r.byte()?; + let profile = r.byte()?; + let conditional = match r.byte()? { + 0 => false, + 1 => true, + _ => return Err(Error::Decode), + }; + let program = match r.byte()? { + 0 => false, + 1 => true, + _ => return Err(Error::Decode), + }; + let count = r.uleb()?; + // Each length takes at least one byte, which bounds the allocation. + if count > data.len() as u64 { + return Err(Error::Decode); + } + let mut object_lengths = Vec::with_capacity(count as usize); + let mut output_len = 0u64; + for _ in 0..count { + let n = r.uleb()?; + output_len = output_len.checked_add(n).ok_or(Error::Overflow)?; + object_lengths.push(n); + } + let body_len_at = r.at; + let body_len = r.uleb()?; + if !program && body_len != output_len { + return Err(Error::Decode); + } + let basis = if conditional { + Some(r.take(BASIS_LEN)?.try_into().expect("32 bytes")) + } else { + None + }; + let body = r.take(usize::try_from(body_len).map_err(|_| Error::Decode)?)?; + let sealed = r.at; + let digest: [u8; DIGEST_LEN] = r.take(DIGEST_LEN)?.try_into().expect("32 bytes"); + let end = r.at; + let representation = if !program { + Representation::Raw + } else if body.starts_with(kernel_prefix()) { + Representation::UlzKernel + } else { + Representation::Program + }; + let info = Info { + format, + profile, + basis, + representation, + object_lengths, + output_len, + body_len, + archive_len: end as u64, + digest, + integrity_ok: chain_digest(&data[..sealed]) == digest, + }; + Ok(( + info, + Layout { + body_len_at, + sealed, + end, + }, + )) +} + +/// Read one archive's frame header and check its integrity seal, without +/// decoding. The bytes must be exactly one archive. +pub fn inspect(archive: &[u8]) -> Result { + let (info, layout) = parse(archive)?; + if layout.end != archive.len() { + return Err(Error::Decode); + } + Ok(info) +} + +/// Split a stream of archives written back to back into its archives. A +/// single archive is a stream of one; an empty stream has none. +pub fn frames(stream: &[u8]) -> Result, Error> { + let mut out = Vec::new(); + let mut rest = stream; + while !rest.is_empty() { + let (_, layout) = parse(rest)?; + let (frame, tail) = rest.split_at(layout.end); + out.push(frame); + rest = tail; + } + Ok(out) +} + +/// The same archive with its output split into `lengths` instead, resealed. +/// Only the object lengths and the seal change: the body, and so the bytes it +/// produces, stay as they are, which is why the lengths must keep their sum. +/// The original seal must be intact. +pub(crate) fn relabel(archive: &[u8], lengths: &[u64]) -> Result, Error> { + let (info, layout) = parse(archive)?; + let sum = lengths + .iter() + .try_fold(0u64, |a, &n| a.checked_add(n)) + .ok_or(Error::Overflow)?; + if layout.end != archive.len() || !info.integrity_ok || sum != info.output_len { + return Err(Error::Decode); + } + let mut out = archive[..8].to_vec(); + push_uleb(&mut out, lengths.len() as u64); + for &n in lengths { + push_uleb(&mut out, n); + } + out.extend_from_slice(&archive[layout.body_len_at..layout.sealed]); + let digest = chain_digest(&out); + out.extend_from_slice(&digest); + Ok(out) +} + +fn push_uleb(out: &mut Vec, mut n: u64) { + loop { + let byte = (n & 0x7f) as u8; + n >>= 7; + if n == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} diff --git a/crates/uorc/src/lib.rs b/crates/uorc/src/lib.rs index 6759512c..816321af 100644 --- a/crates/uorc/src/lib.rs +++ b/crates/uorc/src/lib.rs @@ -6,6 +6,11 @@ use std::fmt; +mod frame; +pub mod tree; + +pub use frame::{frames, inspect, Info, Representation}; + /// Search steps spent by [`compress`] when the caller names no budget. With /// none, compress takes the smaller of the ulz kernel archive and the raw one; /// a positive budget also runs the program search and keeps the smallest. @@ -23,6 +28,8 @@ pub enum Error { Overflow, /// The bytes are not a self-contained UORC archive this client decodes. Decode, + /// The archive decodes, but not to the original it was checked against. + Mismatch, } impl fmt::Display for Error { @@ -31,6 +38,7 @@ impl fmt::Display for Error { Error::Budget => "no checked archive was found within the step budget", Error::Overflow => "a natural-number operation overflowed 64 bits", Error::Decode => "input is not a decodable self-contained UORC archive", + Error::Mismatch => "the archive does not decode to the given original", }) } } @@ -81,8 +89,65 @@ fn compress_here(input: &[u8], steps: u64) -> Result, Error> { /// whose program is the ulz kernel, with a payload the ulz encoder writes, /// runs the certified decoder directly; every other archive is reconstructed /// by the reference semantics. +/// +/// An archive of several ordered objects decodes to their concatenation (see +/// [`decompress_objects`] to get them apart). pub fn decompress(archive: &[u8], work: u64) -> Result, Error> { - on_stack(|| decompress_here(archive, work)) + let info = inspect(archive)?; + if info.object_lengths.len() == 1 { + return on_stack(|| decompress_here(archive, work)); + } + // The certified fast path reads one-object frames. The same body under + // the single total length produces the same bytes; the seal of the + // archive as given is checked before it is relabelled. + let single = frame::relabel(archive, &[info.output_len])?; + on_stack(|| decompress_here(&single, work)) +} + +/// Compress an ordered collection of byte objects into one archive whose +/// header records each object's length. The body codes the objects' +/// concatenation, so similar objects compress together. Any number of objects +/// is allowed, including none and empty ones. +pub fn compress_objects>(objects: &[T], steps: u64) -> Result, Error> { + let lengths: Vec = objects.iter().map(|o| o.as_ref().len() as u64).collect(); + let joined = objects.iter().fold(Vec::new(), |mut all, o| { + all.extend_from_slice(o.as_ref()); + all + }); + let archive = compress(&joined, steps)?; + if lengths.len() == 1 { + Ok(archive) + } else { + frame::relabel(&archive, &lengths) + } +} + +/// Reconstruct the ordered objects of an archive, split at the lengths its +/// header records. +pub fn decompress_objects(archive: &[u8], work: u64) -> Result>, Error> { + let lengths = inspect(archive)?.object_lengths; + let mut output = decompress(archive, work)?.into_iter(); + let objects: Vec> = lengths + .iter() + .map(|&n| output.by_ref().take(n as usize).collect()) + .collect(); + if objects.iter().map(Vec::len).sum::() as u64 != lengths.iter().sum::() + || output.next().is_some() + { + return Err(Error::Decode); + } + Ok(objects) +} + +/// Compress a file tree (see [`tree`]) into one archive. +pub fn compress_tree(entries: &[tree::Entry], steps: u64) -> Result, Error> { + compress_objects(&tree::to_objects(entries)?, steps) +} + +/// Reconstruct the file tree an archive from [`compress_tree`] stores. Every +/// path is checked to be relative and free of `..`. +pub fn decompress_tree(archive: &[u8], work: u64) -> Result, Error> { + tree::from_objects(decompress_objects(archive, work)?) } fn decompress_here(archive: &[u8], work: u64) -> Result, Error> { @@ -93,6 +158,43 @@ fn decompress_here(archive: &[u8], work: u64) -> Result, Error> { } } +/// Check an archive end to end without keeping its output: the frame and the +/// integrity seal must be intact, it must decode, its output must have the +/// lengths the header declares and, when `original` is given, equal it. +/// +/// `stream` may be several archives written back to back; each is checked and +/// `original` is compared with their concatenated output. +pub fn verify(stream: &[u8], original: Option<&[u8]>, work: u64) -> Result, Error> { + let mut infos = Vec::new(); + let mut at = 0usize; + for archive in frames(stream)? { + let info = inspect(archive)?; + if !info.integrity_ok { + return Err(Error::Decode); + } + let output = decompress(archive, work)?; + if output.len() as u64 != info.output_len { + return Err(Error::Decode); + } + if let Some(original) = original { + let end = at.checked_add(output.len()).ok_or(Error::Mismatch)?; + if original.get(at..end) != Some(output.as_slice()) { + return Err(Error::Mismatch); + } + at = end; + } + infos.push(info); + } + if original.is_some_and(|original| original.len() != at) || infos.is_empty() { + return Err(if infos.is_empty() { + Error::Decode + } else { + Error::Mismatch + }); + } + Ok(infos) +} + #[cfg(test)] mod tests { use super::*; @@ -177,6 +279,65 @@ mod tests { ); } + #[test] + fn ordered_objects_round_trip() { + let text: Vec = b"objects share one body. ".repeat(200); + for objects in [ + vec![], + vec![Vec::new()], + vec![Vec::new(), Vec::new(), Vec::new()], + vec![ + text.clone(), + Vec::new(), + (0..=255u8).collect(), + text.clone(), + ], + ] { + let archive = compress_objects(&objects, DEFAULT_STEPS).expect("archive"); + let info = inspect(&archive).expect("header"); + assert!(info.integrity_ok); + assert_eq!(info.object_lengths.len(), objects.len()); + assert_eq!( + decompress_objects(&archive, DEFAULT_DECODE_WORK).expect("decodes"), + objects + ); + verify(&archive, Some(&objects.concat()), DEFAULT_DECODE_WORK).expect("verifies"); + } + } + + #[test] + fn relabelled_archive_with_a_broken_seal_is_rejected() { + let mut archive = compress_objects(&[b"ab".to_vec(), b"cd".to_vec()], 0).expect("archive"); + let last = archive.len() - 1; + archive[last] ^= 1; + assert_eq!( + decompress(&archive, DEFAULT_DECODE_WORK), + Err(Error::Decode) + ); + } + + #[test] + fn streams_verify_frame_by_frame() { + let mut stream = compress(b"first frame ", 0).expect("archive"); + stream.extend(compress(b"second frame", 0).expect("archive")); + assert_eq!(frames(&stream).expect("frames").len(), 2); + let infos = verify( + &stream, + Some(b"first frame second frame"), + DEFAULT_DECODE_WORK, + ) + .expect("verifies"); + assert_eq!(infos.len(), 2); + assert_eq!( + verify( + &stream, + Some(b"first frame second frame!"), + DEFAULT_DECODE_WORK + ), + Err(Error::Mismatch) + ); + } + #[test] fn garbage_is_a_decode_error() { assert_eq!( diff --git a/crates/uorc/src/tree.rs b/crates/uorc/src/tree.rs new file mode 100644 index 00000000..254b3b8f --- /dev/null +++ b/crates/uorc/src/tree.rs @@ -0,0 +1,212 @@ +//! File trees as ordered UORC objects. +//! +//! A tree of `n > 0` entries is stored as the objects +//! `[manifest, file₁, …, fileₖ]`, one object per regular file in manifest +//! order. An empty tree is the archive of zero objects. A one-object archive is +//! always a single plain byte stream, never a tree, so the two never collide. +//! +//! The manifest is: +//! +//! ```text +//! "uorc-tree" 0x01 uleb(entryCount) +//! entry* = kind[1] uleb(pathLength) path -- kind 0 file, 1 file +x, 2 directory +//! ``` +//! +//! Paths are relative, `/`-separated UTF-8, with no empty, `.` or `..` +//! components, so restoring an archive never writes outside its destination. + +use crate::Error; + +const MAGIC: &[u8] = b"uorc-tree\x01"; + +/// What a tree entry is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// A regular file. + File, + /// A regular file with its executable bit set. + Executable, + /// A directory (listed so empty directories survive). + Directory, +} + +/// One entry of a tree: a safe relative path and, for files, its bytes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + /// Relative `/`-separated path. + pub path: String, + /// What the entry is. + pub kind: Kind, + /// File contents; empty for directories. + pub data: Vec, +} + +/// Whether `path` is a relative path that stays inside its destination. +pub fn is_safe_path(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && !path.contains('\\') + && !path.contains('\0') + && !path.contains(':') + && path + .split('/') + .all(|part| !part.is_empty() && part != "." && part != "..") +} + +fn push_uleb(out: &mut Vec, mut n: u64) { + loop { + let byte = (n & 0x7f) as u8; + n >>= 7; + if n == 0 { + out.push(byte); + return; + } + out.push(byte | 0x80); + } +} + +fn read_uleb(data: &[u8], at: &mut usize) -> Result { + let (mut value, mut shift) = (0u64, 0u32); + loop { + let b = *data.get(*at).ok_or(Error::Decode)?; + *at += 1; + if shift >= 64 { + return Err(Error::Decode); + } + value |= u64::from(b & 0x7f) << shift; + if b & 0x80 == 0 { + return Ok(value); + } + shift += 7; + } +} + +/// The ordered objects that store `entries`. Paths must be safe and unique. +pub fn to_objects(entries: &[Entry]) -> Result>, Error> { + if entries.is_empty() { + return Ok(Vec::new()); + } + let mut seen = std::collections::HashSet::new(); + let mut manifest = MAGIC.to_vec(); + push_uleb(&mut manifest, entries.len() as u64); + let mut objects = vec![Vec::new()]; + for entry in entries { + if !is_safe_path(&entry.path) || !seen.insert(entry.path.as_str()) { + return Err(Error::Decode); + } + manifest.push(match entry.kind { + Kind::File => 0, + Kind::Executable => 1, + Kind::Directory => 2, + }); + push_uleb(&mut manifest, entry.path.len() as u64); + manifest.extend_from_slice(entry.path.as_bytes()); + if entry.kind != Kind::Directory { + objects.push(entry.data.clone()); + } + } + objects[0] = manifest; + Ok(objects) +} + +/// The entries the ordered `objects` store, checking every path is safe. +pub fn from_objects(mut objects: Vec>) -> Result, Error> { + if objects.is_empty() { + return Ok(Vec::new()); + } + let manifest = std::mem::take(&mut objects[0]); + let mut files = objects.into_iter().skip(1); + let rest = manifest.strip_prefix(MAGIC).ok_or(Error::Decode)?; + let mut at = 0; + let count = read_uleb(rest, &mut at)?; + let mut entries = Vec::new(); + let mut seen = std::collections::HashSet::new(); + for _ in 0..count { + let kind = match rest.get(at).ok_or(Error::Decode)? { + 0 => Kind::File, + 1 => Kind::Executable, + 2 => Kind::Directory, + _ => return Err(Error::Decode), + }; + at += 1; + let len = usize::try_from(read_uleb(rest, &mut at)?).map_err(|_| Error::Decode)?; + let end = at.checked_add(len).ok_or(Error::Decode)?; + let path = std::str::from_utf8(rest.get(at..end).ok_or(Error::Decode)?) + .map_err(|_| Error::Decode)? + .to_owned(); + at = end; + if !is_safe_path(&path) || !seen.insert(path.clone()) { + return Err(Error::Decode); + } + let data = if kind == Kind::Directory { + Vec::new() + } else { + files.next().ok_or(Error::Decode)? + }; + entries.push(Entry { path, kind, data }); + } + if at != rest.len() || files.next().is_some() { + return Err(Error::Decode); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsafe_paths_are_refused() { + for bad in [ + "", + "/etc/passwd", + "../x", + "a/../b", + "a//b", + "./a", + "a/", + "c:x", + "a\\b", + ] { + assert!(!is_safe_path(bad), "{bad:?}"); + } + assert!(is_safe_path("a/b/c.txt")); + } + + #[test] + fn manifest_round_trip() { + let entries = vec![ + Entry { + path: "d".into(), + kind: Kind::Directory, + data: vec![], + }, + Entry { + path: "d/empty".into(), + kind: Kind::File, + data: vec![], + }, + Entry { + path: "run.sh".into(), + kind: Kind::Executable, + data: b"#!/bin/sh\n".to_vec(), + }, + ]; + let objects = to_objects(&entries).unwrap(); + assert_eq!(objects.len(), 3); + assert_eq!(from_objects(objects).unwrap(), entries); + assert!(to_objects(&[]).unwrap().is_empty()); + assert!(from_objects(vec![]).unwrap().is_empty()); + } + + #[test] + fn hostile_manifest_is_rejected() { + let mut manifest = MAGIC.to_vec(); + manifest.extend_from_slice(&[1, 0, 5]); + manifest.extend_from_slice(b"../up"); + assert_eq!( + from_objects(vec![manifest, b"x".to_vec()]), + Err(Error::Decode) + ); + } +} diff --git a/crates/uorc/tests/cli.rs b/crates/uorc/tests/cli.rs new file mode 100644 index 00000000..22526a82 --- /dev/null +++ b/crates/uorc/tests/cli.rs @@ -0,0 +1,216 @@ +//! End-to-end tests of the `uorc` binary: commands, exit codes and safe output. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +fn uorc(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_uorc")) + .args(args) + .output() + .expect("run uorc") +} + +fn code(output: &Output) -> i32 { + output.status.code().expect("exit code") +} + +/// A fresh scratch directory per test under the target directory. +fn scratch(name: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(name); + let _ = fs::remove_dir_all(&dir); + fs::create_dir_all(&dir).expect("scratch dir"); + dir +} + +fn s(path: &Path) -> &str { + path.to_str().expect("utf-8 path") +} + +/// Every entry in `dir` (temporary files would show up here). +fn entries(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .expect("read dir") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +const TEXT: &[u8] = b"the quick brown fox jumps over the lazy dog. the quick brown fox. "; + +#[test] +fn existing_output_is_refused_and_force_replaces_it() { + let dir = scratch("refuse"); + let input = dir.join("a.txt"); + let archive = dir.join("a.txt.uorc"); + fs::write(&input, TEXT).unwrap(); + fs::write(&archive, b"keep me").unwrap(); + + let refused = uorc(&["compress", s(&input)]); + assert_eq!(code(&refused), 3, "{refused:?}"); + assert_eq!(fs::read(&archive).unwrap(), b"keep me"); + + let forced = uorc(&["compress", "--force", s(&input)]); + assert_eq!(code(&forced), 0, "{forced:?}"); + assert_ne!(fs::read(&archive).unwrap(), b"keep me"); + assert_eq!(entries(&dir), ["a.txt", "a.txt.uorc"]); + + let restored = dir.join("b.txt"); + assert_eq!( + code(&uorc(&["decompress", s(&archive), "-o", s(&restored)])), + 0 + ); + assert_eq!(fs::read(&restored).unwrap(), TEXT); +} + +#[test] +fn failed_run_leaves_no_output() { + let dir = scratch("failed"); + let bad = dir.join("bad.uorc"); + fs::write(&bad, b"UORC not really").unwrap(); + let failed = uorc(&["decompress", s(&bad)]); + assert_eq!(code(&failed), 4, "{failed:?}"); + assert_eq!(entries(&dir), ["bad.uorc"]); +} + +#[test] +fn inspect_reads_the_header_without_decoding() { + let dir = scratch("inspect"); + let input = dir.join("t.txt"); + let text: Vec = TEXT.iter().copied().cycle().take(20_000).collect(); + fs::write(&input, &text).unwrap(); + assert_eq!(code(&uorc(&["compress", s(&input)])), 0); + let archive = dir.join("t.txt.uorc"); + + let shown = uorc(&["inspect", "--json", s(&archive)]); + assert_eq!(code(&shown), 0, "{shown:?}"); + let json = String::from_utf8(shown.stdout).unwrap(); + assert!(json.contains("\"representation\":\"ulz-kernel\""), "{json}"); + assert!(json.contains("\"object_lengths\":[20000]"), "{json}"); + assert!(json.contains("\"integrity_ok\":true"), "{json}"); + + let mut bytes = fs::read(&archive).unwrap(); + let last = bytes.len() - 1; + bytes[last] ^= 1; + let corrupt = dir.join("corrupt.uorc"); + fs::write(&corrupt, &bytes).unwrap(); + assert_eq!(code(&uorc(&["inspect", s(&corrupt)])), 4); + assert_eq!(code(&uorc(&["inspect", s(&input)])), 4); +} + +#[test] +fn verify_checks_integrity_and_the_original() { + let dir = scratch("verify"); + let input = dir.join("v.bin"); + let data: Vec = (0..=255u8).cycle().take(3000).collect(); + fs::write(&input, &data).unwrap(); + assert_eq!(code(&uorc(&["compress", s(&input)])), 0); + let archive = dir.join("v.bin.uorc"); + + assert_eq!(code(&uorc(&["verify", s(&archive)])), 0); + assert_eq!(code(&uorc(&["verify", s(&archive), s(&input)])), 0); + + let other = dir.join("other.bin"); + fs::write(&other, &data[1..]).unwrap(); + assert_eq!(code(&uorc(&["verify", s(&archive), s(&other)])), 7); + assert_eq!(code(&uorc(&["verify", s(&input)])), 4); + assert_eq!(entries(&dir), ["other.bin", "v.bin", "v.bin.uorc"]); +} + +#[test] +fn usage_and_io_errors_have_their_own_codes() { + assert_eq!(code(&uorc(&["frobnicate", "x"])), 2); + assert_eq!(code(&uorc(&["compress", "--steps", "many", "x"])), 2); + assert_eq!(code(&uorc(&["compress", "/nonexistent/uorc/input"])), 3); +} + +/// Every file under `dir`, as (relative path, bytes), sorted. +fn tree(dir: &Path) -> Vec<(String, Vec)> { + fn walk(base: &Path, dir: &Path, out: &mut Vec<(String, Vec)>) { + for entry in fs::read_dir(dir).unwrap() { + let path = entry.unwrap().path(); + let rel = path + .strip_prefix(base) + .unwrap() + .to_string_lossy() + .into_owned(); + if path.is_dir() { + out.push((format!("{rel}/"), Vec::new())); + walk(base, &path, out); + } else { + out.push((rel, fs::read(&path).unwrap())); + } + } + } + let mut out = Vec::new(); + walk(dir, dir, &mut out); + out.sort(); + out +} + +#[test] +fn directories_round_trip_with_empty_files_binary_and_nesting() { + let dir = scratch("tree"); + let src = dir.join("src"); + fs::create_dir_all(src.join("a/b/c")).unwrap(); + fs::create_dir_all(src.join("empty-dir")).unwrap(); + fs::write(src.join("a/b/c/deep.txt"), TEXT.repeat(50)).unwrap(); + fs::write(src.join("a/empty"), b"").unwrap(); + fs::write( + src.join("bin"), + (0..=255u8).cycle().take(5000).collect::>(), + ) + .unwrap(); + fs::write(src.join("a/copy.txt"), TEXT.repeat(50)).unwrap(); + + let archive = dir.join("src.uorc"); + let packed = uorc(&["compress", s(&src)]); + assert_eq!(code(&packed), 0, "{packed:?}"); + let shown = String::from_utf8(uorc(&["inspect", "--json", s(&archive)]).stdout).unwrap(); + assert!(shown.contains("\"objects\":5"), "{shown}"); // manifest + 4 files + + let out = dir.join("out"); + let unpacked = uorc(&["decompress", s(&archive), "-o", s(&out)]); + assert_eq!(code(&unpacked), 0, "{unpacked:?}"); + assert_eq!(tree(&out.join("src")), tree(&src)); + + // Restoring again is refused; --resume finishes a partial restore. + assert_eq!(code(&uorc(&["decompress", s(&archive), "-o", s(&out)])), 3); + fs::remove_file(out.join("src/bin")).unwrap(); + fs::write(out.join("src/a/empty"), b"stale").unwrap(); + let resumed = uorc(&["decompress", "--resume", s(&archive), "-o", s(&out)]); + assert_eq!(code(&resumed), 0, "{resumed:?}"); + assert_eq!(tree(&out.join("src")), tree(&src)); +} + +#[test] +fn several_files_and_an_empty_directory() { + let dir = scratch("several"); + fs::write(dir.join("one"), b"1").unwrap(); + fs::write(dir.join("two"), b"").unwrap(); + fs::create_dir(dir.join("nothing")).unwrap(); + let archive = dir.join("all.uorc"); + let packed = uorc(&[ + "compress", + s(&dir.join("one")), + s(&dir.join("two")), + s(&dir.join("nothing")), + "-o", + s(&archive), + ]); + assert_eq!(code(&packed), 0, "{packed:?}"); + let out = dir.join("out"); + assert_eq!(code(&uorc(&["decompress", s(&archive), "-o", s(&out)])), 0); + assert_eq!(fs::read(out.join("one")).unwrap(), b"1"); + assert_eq!(fs::read(out.join("two")).unwrap(), b""); + assert!(out.join("nothing").is_dir()); + assert_eq!( + code(&uorc(&[ + "compress", + s(&dir.join("one")), + s(&dir.join("two")) + ])), + 2 + ); +} diff --git a/web/app.js b/web/app.js index d25e931c..69db8e1d 100644 --- a/web/app.js +++ b/web/app.js @@ -23,27 +23,103 @@ const tick = () => new Promise((resolve) => setTimeout(resolve, 20)); try { const uorc = await load("./uorc.wasm"); fetch("version.txt").then((r) => r.ok ? r.text() : "").then((v) => { if (v) $("version").textContent = `Release ${v.trim()}`; }); - $("compress").disabled = $("decompress").disabled = false; + for (const id of ["compress", "decompress", "inspect", "verify"]) $(id).disabled = false; status.textContent = "Ready. Choose a file, or run an example below."; - for (const id of ["compress", "decompress"]) { - $(id).addEventListener("click", async () => { - const file = $("file").files[0]; - if (!file) { status.textContent = "Choose a file first."; return; } - const input = new Uint8Array(await file.arrayBuffer()); - status.textContent = `${id === "compress" ? "Compressing" : "Decompressing"} ${file.name}…`; - await tick(); - const started = performance.now(); - try { - const output = id === "compress" ? uorc.compress(input, steps()) : uorc.decompress(input); - const seconds = ((performance.now() - started) / 1000).toFixed(2); - status.textContent = `${input.length} → ${output.length} bytes in ${seconds} s.`; - offer(output, id === "compress" ? `${file.name}.uorc` : file.name.replace(/\.uorc$/, "") || "output"); - } catch (error) { - status.textContent = `Failed: ${error.message}.`; + const bytesOf = async (file) => new Uint8Array(await file.arrayBuffer()); + const report = (text) => { $("report").hidden = !text; $("report").textContent = text ?? ""; }; + const picked = () => { + const folder = [...$("folder").files]; + return folder.length ? folder : [...$("file").files]; + }; + const timed = async (label, work) => { + status.textContent = `${label}…`; + report(); + $("download").replaceChildren(); + await tick(); + const started = performance.now(); + try { + const said = await work(); + status.textContent = `${said} in ${((performance.now() - started) / 1000).toFixed(2)} s.`; + } catch (error) { + status.textContent = `Failed: ${error.message}.`; + } + }; + + $("compress").addEventListener("click", async () => { + const files = picked(); + if (!files.length) { status.textContent = "Choose files or a folder first."; return; } + const tree = files.length > 1 || $("folder").files.length > 0; + await timed(`Compressing ${files.length} file(s)`, async () => { + if (!tree) { + const input = await bytesOf(files[0]); + const output = uorc.compress(input, steps()); + offer(output, `${files[0].name}.uorc`); + return `${input.length} → ${output.length} bytes`; + } + const entries = []; + const dirs = new Set(); + for (const file of files) { + const path = file.webkitRelativePath || file.name; + const parts = path.split("/"); + for (let i = 1; i < parts.length; i++) dirs.add(parts.slice(0, i).join("/")); + entries.push({ path, data: await bytesOf(file) }); } + const all = [...[...dirs].sort().map((path) => ({ path, kind: "directory" })), ...entries]; + const output = uorc.compressFiles(all, steps()); + const root = files[0].webkitRelativePath ? files[0].webkitRelativePath.split("/")[0] : "files"; + offer(output, `${root}.uorc`); + return `${entries.length} files, ${entries.reduce((n, e) => n + e.data.length, 0)} → ${output.length} bytes`; }); - } + }); + + $("decompress").addEventListener("click", async () => { + const file = $("file").files[0]; + if (!file) { status.textContent = "Choose an archive first."; return; } + await timed(`Decompressing ${file.name}`, async () => { + const archive = await bytesOf(file); + if (uorc.inspect(archive).objects === 1) { + const output = uorc.decompress(archive); + offer(output, file.name.replace(/\.uorc$/, "") || "output"); + return `${archive.length} → ${output.length} bytes`; + } + const entries = uorc.decompressFiles(archive); + const links = entries.filter((e) => e.kind !== "directory").map((e) => { + const link = document.createElement("a"); + link.href = URL.createObjectURL(new Blob([e.data])); + link.download = e.path.split("/").pop(); + link.textContent = `${e.path} (${e.data.length} bytes)`; + const item = document.createElement("li"); + item.append(link); + return item; + }); + const list = document.createElement("ul"); + list.append(...links); + $("download").replaceChildren(list); + return `${entries.length} entries restored`; + }); + }); + + $("inspect").addEventListener("click", async () => { + const file = $("file").files[0]; + if (!file) { status.textContent = "Choose an archive first."; return; } + await timed(`Inspecting ${file.name}`, async () => { + const info = uorc.inspect(await bytesOf(file)); + report(JSON.stringify(info, null, 2)); + return info.integrity_ok ? "Header read, integrity ok" : "Header read, integrity MISMATCH"; + }); + }); + + $("verify").addEventListener("click", async () => { + const file = $("file").files[0]; + if (!file) { status.textContent = "Choose an archive first."; return; } + const original = $("original").files[0]; + await timed(`Verifying ${file.name}`, async () => { + const infos = uorc.verify(await bytesOf(file), original ? await bytesOf(original) : undefined); + report(JSON.stringify(infos, null, 2)); + return original ? `Verified: identical to ${original.name}` : "Verified: decodes and integrity holds"; + }); + }); const body = $("example-results").querySelector("tbody"); for (const [label, make] of examples) { diff --git a/web/index.html b/web/index.html index b33a578d..78a90e10 100644 --- a/web/index.html +++ b/web/index.html @@ -19,8 +19,16 @@

uorc

Try it

- - + + +
+
+ + +
+
+ +
@@ -29,9 +37,16 @@

Try it

+ +

Loading the WebAssembly client…

-

+
+ +

Several files or a folder become one archive of ordered objects: a manifest + of relative paths, then each file. Decompressing it lists every file for + download. Inspect reads the frame header and checks the integrity seal without + decoding; Verify decodes in memory and, given an original, compares the bytes.

@@ -50,16 +65,32 @@

Use it

Command line

uorc compress --stats notes.txt      # writes notes.txt.uorc
 uorc decompress notes.txt.uorc -o notes.restored.txt
-cat data.bin | uorc compress - > data.uorc
+cat data.bin | uorc compress - > data.uorc +uorc compress photos/ # writes photos.uorc (a whole tree) +uorc decompress photos.uorc -o restore/ # --resume finishes an interrupted restore +uorc inspect notes.txt.uorc # header and integrity, no decoding +uorc verify notes.txt.uorc notes.txt # decode, check, compare +

Outputs are written to a temporary file and renamed into place; an existing + file is only replaced with --force. Exit status: 0 ok, 2 usage, + 3 I/O, 4 invalid archive or integrity, 5 step budget, 6 overflow, 7 differs + from the original.

Rust library

let archive = uorc::compress(&bytes, uorc::DEFAULT_STEPS)?;
 let original = uorc::decompress(&archive, uorc::DEFAULT_DECODE_WORK)?;
-assert_eq!(original, bytes);
+assert_eq!(original, bytes); +let info = uorc::inspect(&archive)?; // header + integrity, no decode +uorc::verify(&archive, Some(&bytes), uorc::DEFAULT_DECODE_WORK)?; +let many = uorc::compress_objects(&[a, b, c], uorc::DEFAULT_STEPS)?; +let parts = uorc::decompress_objects(&many, uorc::DEFAULT_DECODE_WORK)?;

JavaScript (this page's module)

import { load } from "./uorc.js";
 const uorc = await load("./uorc.wasm");
 const archive = uorc.compress(new TextEncoder().encode("hello hello hello"));
-const original = uorc.decompress(archive);
+const original = uorc.decompress(archive); +uorc.inspect(archive); // { representation, object_lengths, integrity_ok, ... } +uorc.verify(archive, original); // throws if it does not reproduce original +const bundle = uorc.compressFiles([{ path: "a.txt", data }, { path: "d", kind: "directory" }]); +const files = uorc.decompressFiles(bundle); // [{ path, kind, data }]

Binaries, the library source bundle and this WebAssembly module are attached to each GitHub release. Benchmark results for the release are on the benchmarks page.

diff --git a/web/uorc.js b/web/uorc.js index d55f1314..765d757f 100644 --- a/web/uorc.js +++ b/web/uorc.js @@ -1,20 +1,72 @@ // Thin loader for the uorc WebAssembly client (crates/uorc-wasm). const ERRORS = ["", "no checked archive was found within the step budget", - "a natural-number operation overflowed 64 bits", "input is not a decodable self-contained UORC archive"]; + "a natural-number operation overflowed 64 bits", "input is not a decodable self-contained UORC archive", + "the archive does not decode to the given original"]; +const KINDS = ["file", "executable", "directory"]; + +// Entries cross into wasm as u32 count, then per entry u8 kind, u32 path length, +// path, u32 data length, data (little-endian). +function packEntries(entries) { + const enc = new TextEncoder(); + const paths = entries.map((e) => enc.encode(e.path)); + const size = 4 + entries.reduce((n, e, i) => n + 9 + paths[i].length + (e.data?.length ?? 0), 0); + const out = new Uint8Array(size); + const view = new DataView(out.buffer); + let at = 0; + view.setUint32(at, entries.length, true); at += 4; + entries.forEach((e, i) => { + const data = e.data ?? new Uint8Array(0); + out[at++] = Math.max(0, KINDS.indexOf(e.kind ?? "file")); + view.setUint32(at, paths[i].length, true); at += 4; out.set(paths[i], at); at += paths[i].length; + view.setUint32(at, data.length, true); at += 4; out.set(data, at); at += data.length; + }); + return out; +} + +function unpackEntries(bytes) { + const dec = new TextDecoder(); + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + let at = 0; + const count = view.getUint32(at, true); at += 4; + const entries = []; + for (let i = 0; i < count; i++) { + const kind = KINDS[bytes[at++]]; + const pathLength = view.getUint32(at, true); at += 4; + const path = dec.decode(bytes.subarray(at, at + pathLength)); at += pathLength; + const dataLength = view.getUint32(at, true); at += 4; + entries.push({ path, kind, data: bytes.slice(at, at + dataLength) }); at += dataLength; + } + return entries; +} export async function load(url) { const response = await fetch(url); const { instance } = await WebAssembly.instantiateStreaming(response, {}); const x = instance.exports; - const call = (bytes, op) => { - const at = x.input_buffer(bytes.length); + const put = (alloc, bytes) => { + const at = alloc(bytes.length); // may grow memory, so read the buffer after new Uint8Array(x.memory.buffer, at, bytes.length).set(bytes); + }; + const call = (bytes, op) => { + put(x.input_buffer, bytes); const code = op(); if (code !== 0) throw new Error(ERRORS[code] ?? `uorc error ${code}`); return new Uint8Array(x.memory.buffer, x.output_ptr(), x.output_len()).slice(); }; + const json = (bytes) => JSON.parse(new TextDecoder().decode(bytes)); return { compress: (bytes, steps = 0n) => call(bytes, () => x.compress(BigInt(steps))), decompress: (archive) => call(archive, () => x.decompress()), + // Frame header and integrity seal of one archive, without decoding. + inspect: (archive) => json(call(archive, () => x.inspect())), + // Decode in memory and check; with an original, also compare. Returns one + // description per archive in the stream; throws on any failure. + verify: (archive, original) => json(call(archive, () => { + if (original) put(x.original_buffer, original); + return x.verify(original ? 1 : 0); + })), + // entries: [{ path: "dir/a.txt", kind: "file" | "executable" | "directory", data: Uint8Array }] + compressFiles: (entries, steps = 0n) => call(packEntries(entries), () => x.compress_tree(BigInt(steps))), + decompressFiles: (archive) => unpackEntries(call(archive, () => x.decompress_tree())), }; }