diff --git a/.gitignore b/.gitignore index f84ace4e..c3cb32df 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ __pycache__/ # wasm-pack build output: pkg/ is the npm package, www/pkg/ the demo site copy /wasm/pkg/ /wasm/www/pkg/ +# cargo-fuzz working dirs: seeds live in fuzz/seeds/ and are checked in +/fuzz/corpus/ +/fuzz/artifacts/ +/fuzz/coverage/ diff --git a/Cargo.lock b/Cargo.lock index a01c9140..b7da1011 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,7 +91,6 @@ dependencies = [ name = "anydoc" version = "0.1.9" dependencies = [ - "calamine", "cfb", "csv", "encoding_rs", @@ -134,16 +133,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "atoi_simd" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cdb3708a128e559a30fb830e8a77a5022ee6902806925c216658652b452a44" -dependencies = [ - "debug_unsafe", - "rustversion", -] - [[package]] name = "autocfg" version = "1.5.1" @@ -195,30 +184,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "calamine" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" -dependencies = [ - "atoi_simd", - "byteorder", - "chrono", - "codepage", - "encoding_rs", - "fast-float2", - "log", - "quick-xml", - "serde", - "zip", -] - [[package]] name = "cbc" version = "0.1.2" @@ -274,7 +239,6 @@ checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", "num-traits", - "serde", "windows-link", ] @@ -288,15 +252,6 @@ dependencies = [ "inout", ] -[[package]] -name = "codepage" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4" -dependencies = [ - "encoding_rs", -] - [[package]] name = "colorchoice" version = "1.0.5" @@ -433,12 +388,6 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d83cb7e7a873830708d6b02a78cd36a592c6fa14bf267b68725103b85c0d77f" -[[package]] -name = "debug_unsafe" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eed2c4702fa172d1ce21078faa7c5203e69f5394d48cc436d25928394a867a2" - [[package]] name = "defmt" version = "1.1.1" @@ -566,12 +515,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "fast-float2" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" - [[package]] name = "fastrand" version = "2.5.0" @@ -1186,7 +1129,6 @@ version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ - "encoding_rs", "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 04ed82f8..cca4e47d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ exclude = ["fuzz"] name = "anydoc" version = "0.1.9" edition = "2024" -# Edition 2024 needs 1.85; zip and calamine both raise it to 1.88. +# Edition 2024 needs 1.85; zip raises it to 1.88. rust-version = "1.88" description = "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" license = "MIT" @@ -22,7 +22,6 @@ insta = "1" sha2 = "0.11" [dependencies] -calamine = { version = "0.36.1", features = ["dates"] } cfb = "0.14.0" csv = "1.4.0" flate2 = "1" diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 09c8896e..b8ea2618 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,6 +10,8 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" anydoc = { path = ".." } +cfb = "0.14.0" +zip = { version = "8.6.0", default-features = false, features = ["deflate"] } [[bin]] name = "docx" @@ -64,3 +66,21 @@ name = "pdf" path = "fuzz_targets/pdf.rs" test = false doc = false + +[[bin]] +name = "xls" +path = "fuzz_targets/xls.rs" +test = false +doc = false + +[[bin]] +name = "xlsb" +path = "fuzz_targets/xlsb.rs" +test = false +doc = false + +[[bin]] +name = "numfmt" +path = "fuzz_targets/numfmt.rs" +test = false +doc = false diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 00000000..9d84efff --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,12 @@ +# Fuzzing + + cargo +nightly fuzz run xlsx fuzz/corpus/xlsx fuzz/seeds/xlsx + +`fuzz/seeds/` holds checked-in starting inputs; `fuzz/corpus/` is the working +directory libfuzzer writes to and is not checked in. + +`xls`, `xlsb` and `numfmt` wrap their input in a valid container (an OLE +compound file, an OPC package, a styles part) so mutation reaches the record +and format-code parsers rather than dying at the container gate. `xlsx` takes +a whole workbook, and its seeds cover all three containers because the +frontend picks the reader from the bytes. diff --git a/fuzz/fuzz_targets/numfmt.rs b/fuzz/fuzz_targets/numfmt.rs new file mode 100644 index 00000000..3b10040a --- /dev/null +++ b/fuzz/fuzz_targets/numfmt.rs @@ -0,0 +1,54 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use std::io::{Cursor, Write}; + +// A number format code, carried into the engine on a minimal workbook. The +// grammar is where the parsing complexity lives, and reaching it through a +// discovered styles part would cost the fuzzer most of its budget. Only the +// control characters XML forbids are dropped, so whitespace a code may carry +// still reaches the parser. +fuzz_target!(|data: &[u8]| { + let code = String::from_utf8_lossy(data); + let escaped: String = code + .chars() + .filter(|c| !c.is_control() || matches!(c, '\t' | '\n' | '\r')) + .map(|c| match c { + '&' => "&".to_string(), + '<' => "<".to_string(), + '>' => ">".to_string(), + '"' => """.to_string(), + c => c.to_string(), + }) + .collect(); + let styles = format!( + r#""# + ); + + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + let parts: [(&str, &str); 6] = [ + ("[Content_Types].xml", CONTENT_TYPES), + ("_rels/.rels", RELS), + ("xl/workbook.xml", WORKBOOK), + ("xl/_rels/workbook.xml.rels", WORKBOOK_RELS), + ("xl/styles.xml", &styles), + ("xl/worksheets/sheet1.xml", SHEET), + ]; + for (name, body) in parts { + if zip.start_file(name, opts).is_err() || zip.write_all(body.as_bytes()).is_err() { + return; + } + } + let Ok(bytes) = zip.finish() else { + return; + }; + let _ = anydoc::to_markdown_bytes(bytes.into_inner().as_slice(), anydoc::Format::Excel); +}); + +const SML: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; +const CONTENT_TYPES: &str = r#""#; +const RELS: &str = r#""#; +const WORKBOOK: &str = r#""#; +const WORKBOOK_RELS: &str = r#""#; +const SHEET: &str = r#"1234.50.075text"#; diff --git a/fuzz/fuzz_targets/xls.rs b/fuzz/fuzz_targets/xls.rs new file mode 100644 index 00000000..e0ffe43d --- /dev/null +++ b/fuzz/fuzz_targets/xls.rs @@ -0,0 +1,22 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use std::io::{Cursor, Write}; + +// The BIFF record stream, wrapped in a valid OLE container so mutation +// reaches the reader instead of dying at the container gate. +fuzz_target!(|data: &[u8]| { + let Ok(mut ole) = cfb::CompoundFile::create(Cursor::new(Vec::new())) else { + return; + }; + match ole.create_stream("Workbook") { + Ok(mut stream) => { + if stream.write_all(data).is_err() { + return; + } + } + Err(_) => return, + } + let bytes = ole.into_inner().into_inner(); + let _ = anydoc::to_markdown_bytes(&bytes, anydoc::Format::Excel); +}); diff --git a/fuzz/fuzz_targets/xlsb.rs b/fuzz/fuzz_targets/xlsb.rs new file mode 100644 index 00000000..4c07eb9a --- /dev/null +++ b/fuzz/fuzz_targets/xlsb.rs @@ -0,0 +1,40 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use std::io::{Cursor, Write}; + +// The worksheet, styles and shared-string parts, wrapped in a package whose +// workbook part and relationships are already valid. Fuzzing the workbook +// part instead would strand the input there: without a resolvable worksheet +// relationship every sheet is skipped, and the cell, format and string +// readers never run. The workbook part is parsed by the same record reader +// this does reach. +fuzz_target!(|data: &[u8]| { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + let parts: [(&str, &[u8]); 8] = [ + ("[Content_Types].xml", CONTENT_TYPES.as_bytes()), + ("_rels/.rels", RELS.as_bytes()), + ("xl/workbook.bin", WORKBOOK), + ("xl/_rels/workbook.bin.rels", WORKBOOK_RELS.as_bytes()), + ("xl/worksheets/sheet1.bin", data), + ("xl/worksheets/sheet2.bin", data), + ("xl/styles.bin", data), + ("xl/sharedStrings.bin", data), + ]; + for (name, body) in parts { + if zip.start_file(name, opts).is_err() || zip.write_all(body).is_err() { + return; + } + } + let Ok(bytes) = zip.finish() else { + return; + }; + let _ = anydoc::to_markdown_bytes(bytes.into_inner().as_slice(), anydoc::Format::Excel); +}); + +const WORKBOOK: &[u8] = include_bytes!("../seeds/xlsb/workbook-bin"); + +const CONTENT_TYPES: &str = r#""#; +const RELS: &str = r#""#; +const WORKBOOK_RELS: &str = r#""#; diff --git a/fuzz/seeds/numfmt/ampm-multibyte b/fuzz/seeds/numfmt/ampm-multibyte new file mode 100644 index 00000000..1cc7e92e --- /dev/null +++ b/fuzz/seeds/numfmt/ampm-multibyte @@ -0,0 +1 @@ +aaaaé \ No newline at end of file diff --git a/fuzz/seeds/numfmt/ampm-token-multibyte b/fuzz/seeds/numfmt/ampm-token-multibyte new file mode 100644 index 00000000..6e3bfe35 --- /dev/null +++ b/fuzz/seeds/numfmt/ampm-token-multibyte @@ -0,0 +1 @@ +h:mm AM/PMé \ No newline at end of file diff --git a/fuzz/seeds/numfmt/conditions b/fuzz/seeds/numfmt/conditions new file mode 100644 index 00000000..7b0d7157 --- /dev/null +++ b/fuzz/seeds/numfmt/conditions @@ -0,0 +1 @@ +[>100]0.0;[<=100]0.00 \ No newline at end of file diff --git a/fuzz/seeds/numfmt/currency b/fuzz/seeds/numfmt/currency new file mode 100644 index 00000000..4f65ad99 --- /dev/null +++ b/fuzz/seeds/numfmt/currency @@ -0,0 +1 @@ +"$"#,##0.00 \ No newline at end of file diff --git a/fuzz/seeds/numfmt/datetime b/fuzz/seeds/numfmt/datetime new file mode 100644 index 00000000..15fe68a4 --- /dev/null +++ b/fuzz/seeds/numfmt/datetime @@ -0,0 +1 @@ +yyyy-mm-dd hh:mm:ss AM/PM \ No newline at end of file diff --git a/fuzz/seeds/numfmt/elapsed b/fuzz/seeds/numfmt/elapsed new file mode 100644 index 00000000..516b83ad --- /dev/null +++ b/fuzz/seeds/numfmt/elapsed @@ -0,0 +1 @@ +[h]:mm:ss \ No newline at end of file diff --git a/fuzz/seeds/numfmt/four-sections b/fuzz/seeds/numfmt/four-sections new file mode 100644 index 00000000..26719ef3 --- /dev/null +++ b/fuzz/seeds/numfmt/four-sections @@ -0,0 +1 @@ +#,##0.00;[Red](#,##0.00);"nil";@ \ No newline at end of file diff --git a/fuzz/seeds/numfmt/fraction b/fuzz/seeds/numfmt/fraction new file mode 100644 index 00000000..e984e197 --- /dev/null +++ b/fuzz/seeds/numfmt/fraction @@ -0,0 +1 @@ +# ??/?? \ No newline at end of file diff --git a/fuzz/seeds/numfmt/percent b/fuzz/seeds/numfmt/percent new file mode 100644 index 00000000..7e95c3cc --- /dev/null +++ b/fuzz/seeds/numfmt/percent @@ -0,0 +1 @@ +0.0% \ No newline at end of file diff --git a/fuzz/seeds/numfmt/scientific b/fuzz/seeds/numfmt/scientific new file mode 100644 index 00000000..b466a3ff --- /dev/null +++ b/fuzz/seeds/numfmt/scientific @@ -0,0 +1 @@ +0.00E+00 \ No newline at end of file diff --git a/fuzz/seeds/xls/workbook-stream b/fuzz/seeds/xls/workbook-stream new file mode 100644 index 00000000..e9bfb1ca Binary files /dev/null and b/fuzz/seeds/xls/workbook-stream differ diff --git a/fuzz/seeds/xlsb/sheet1-bin b/fuzz/seeds/xlsb/sheet1-bin new file mode 100644 index 00000000..68321558 Binary files /dev/null and b/fuzz/seeds/xlsb/sheet1-bin differ diff --git a/fuzz/seeds/xlsb/styles-bin b/fuzz/seeds/xlsb/styles-bin new file mode 100644 index 00000000..bfcf3e2d Binary files /dev/null and b/fuzz/seeds/xlsb/styles-bin differ diff --git a/fuzz/seeds/xlsb/workbook-bin b/fuzz/seeds/xlsb/workbook-bin new file mode 100644 index 00000000..7354d9f3 Binary files /dev/null and b/fuzz/seeds/xlsb/workbook-bin differ diff --git a/fuzz/seeds/xlsx/merged.xlsx b/fuzz/seeds/xlsx/merged.xlsx new file mode 100644 index 00000000..8dd24429 Binary files /dev/null and b/fuzz/seeds/xlsx/merged.xlsx differ diff --git a/fuzz/seeds/xlsx/sheet.xls b/fuzz/seeds/xlsx/sheet.xls new file mode 100644 index 00000000..6d830f15 Binary files /dev/null and b/fuzz/seeds/xlsx/sheet.xls differ diff --git a/fuzz/seeds/xlsx/sheet.xlsb b/fuzz/seeds/xlsx/sheet.xlsb new file mode 100644 index 00000000..e69c2945 Binary files /dev/null and b/fuzz/seeds/xlsx/sheet.xlsb differ diff --git a/fuzz/seeds/xlsx/sheet.xlsx b/fuzz/seeds/xlsx/sheet.xlsx new file mode 100644 index 00000000..44dba2b5 Binary files /dev/null and b/fuzz/seeds/xlsx/sheet.xlsx differ diff --git a/src/formats/sheet/mod.rs b/src/formats/sheet/mod.rs index f61ebcc6..5df92ad3 100644 --- a/src/formats/sheet/mod.rs +++ b/src/formats/sheet/mod.rs @@ -1,183 +1,85 @@ -//! Excel spreadsheets (xlsx, xlsm, xlsb, xls) via calamine. +//! Excel spreadsheets (xlsx, xlsm, xlsb, xls). Every container is read +//! in-house: SpreadsheetML as XML (xlsx, xlsm) or binary (xlsb), and +//! OLE-based BIFF (xls). All three share the number format engine and grid +//! assembly, so one workbook saved in any of them converts identically. -use crate::error::ConvertError; -use crate::model::{Block, Cell, Document, GridBuilder, Inline, TableKind}; -use crate::shared::header::resolve_header_rows; -use crate::shared::text::clean_text; -use calamine::{Data, Dimensions, Reader, Sheets, open_workbook_auto_from_rs}; -use std::collections::{HashMap, HashSet}; -use std::io::Cursor; +mod numfmt; +mod xls; +mod xlsb; +mod xlsx; -/// Run one calamine operation behind a panic barrier: the calamine fork can -/// panic on corrupt containers (pending an upstream fix), and a dependency -/// panic must degrade to a typed error - while bugs in this crate's own code -/// stay panics. `AssertUnwindSafe` is sound here because a caught panic -/// always propagates as an error, so the workbook is never used again. -fn contained(op: &str, f: impl FnOnce() -> T) -> Result { - std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|_| { - log::warn!("spreadsheet parser panicked during {op} on malformed input"); - ConvertError::malformed("unreadable workbook (parser aborted)") - }) -} +use crate::error::ConvertError; +use crate::model::Document; +use crate::package::archive::probe_ole; +use crate::package::relationships::{read_rels, rel_type}; +use crate::package::{Package, path}; pub fn parse(bytes: &[u8]) -> Result { - let mut workbook = - contained("workbook open", || open_workbook_auto_from_rs(Cursor::new(bytes)))? - .map_err(map_open_error)?; - let sheet_names = contained("sheet listing", || workbook.sheet_names().to_owned())?; - let multi_sheet = sheet_names.len() > 1; - let merged = merged_regions(&mut workbook, &sheet_names)?; - - let mut doc = Document::default(); - let mut failed = 0usize; - for name in &sheet_names { - let range = match contained("worksheet read", || workbook.worksheet_range(name))? { - Ok(r) => r, - Err(e) => { - log::warn!("skipping unreadable sheet {name:?}: {e}"); - failed += 1; - continue; - } + const OLE_MAGIC: [u8; 8] = [0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; + if bytes.starts_with(&OLE_MAGIC) { + // An encrypted OOXML package is an OLE container carrying no BIFF + // workbook stream, so it has to be named before the reader looks + // for one. + return match probe_ole(bytes) { + Some(e @ ConvertError::Encrypted) => Err(e), + _ => xls::parse(bytes), }; - if range.is_empty() { - continue; - } - // Merged regions in range-relative coordinates: the top-left cell - // becomes a spanning origin, the other positions are covered. - let start = range.start().unwrap_or((0, 0)); - let (height, width) = (range.height(), range.width()); - let mut origins: HashMap<(usize, usize), (u32, u32)> = HashMap::new(); - let mut covered: HashSet<(usize, usize)> = HashSet::new(); - for d in merged.get(name.as_str()).map(Vec::as_slice).unwrap_or_default() { - // Intersect the absolute merged region with the used range first: - // a region wholly above or left of the range must not saturate - // onto relative (0,0), and positions outside the range are never - // materialized (a crafted region list must not force insertions - // beyond the cells that actually exist). - let (row0, col0) = (d.start.0.max(start.0), d.start.1.max(start.1)); - let row_end = (d.end.0 as u64 + 1).min(start.0 as u64 + height as u64); - let col_end = (d.end.1 as u64 + 1).min(start.1 as u64 + width as u64); - if (row0 as u64) >= row_end || (col0 as u64) >= col_end { - continue; - } - // Translate the non-empty intersection to range-relative form. - let r0 = (row0 - start.0) as usize; - let c0 = (col0 - start.1) as usize; - let r1 = (row_end - start.0 as u64) as usize; - let c1 = (col_end - start.1 as u64) as usize; - if r1 - r0 == 1 && c1 - c0 == 1 { - continue; - } - origins.insert((r0, c0), ((c1 - c0) as u32, (r1 - r0) as u32)); - for r in r0..r1 { - for c in c0..c1 { - if (r, c) != (r0, c0) { - covered.insert((r, c)); - } - } - } - } - let mut builder = GridBuilder::new(); - for (r, row) in range.rows().enumerate() { - builder.next_row(); - for (c, data) in row.iter().enumerate() { - if covered.contains(&(r, c)) { - builder.covered(); - continue; - } - let text = format_data(data); - let cell = if text.is_empty() { - Cell::default() - } else { - Cell::from_inlines(vec![Inline::plain(text)]) - }; - match origins.get(&(r, c)) { - Some(&(col_span, row_span)) => { - builder.place(Cell::spanning(cell.blocks, col_span, row_span))? - } - None => builder.place(cell)?, - } - } - } - // A spreadsheet marks no header row, so the shape of the data decides. - let mut table = builder.finish(TableKind::Data); - if table.grid.is_empty() { - continue; - } - table.header_rows = resolve_header_rows(&table, 0); - if multi_sheet { - doc.blocks.push(Block::heading(2, vec![Inline::plain(name.clone())])); - } - doc.blocks.push(Block::Table(table)); } - if !sheet_names.is_empty() && failed == sheet_names.len() { - return Err(ConvertError::malformed("no sheet in the workbook could be read")); + // Failing to open as a ZIP means not a workbook; a resource limit + // tripped by a valid archive still propagates. + let mut pkg = match Package::open(bytes) { + Ok(pkg) => pkg, + Err(ConvertError::Malformed { .. }) => return Err(not_a_workbook()), + Err(e) => return Err(e), + }; + let Some(wb_part) = main_part(&mut pkg)? else { + return Err(not_a_workbook()); + }; + match classify(&mut pkg, &wb_part)? { + Some(Container::Xml) => xlsx::parse(&mut pkg, &wb_part), + Some(Container::Bin) => xlsb::parse(&mut pkg, &wb_part), + None => Err(not_a_workbook()), } - Ok(doc) } -/// Merged regions per sheet, where the container format exposes them (xlsx -/// via each worksheet's mergeCells part, xls via BIFF MERGEDCELLS). -fn merged_regions( - workbook: &mut Sheets, - sheet_names: &[String], -) -> Result>, ConvertError> { - let mut out: HashMap> = HashMap::new(); - for name in sheet_names { - let regions = match workbook { - Sheets::Xlsx(x) => { - contained("merged-region listing", || x.merge_cells_by_sheet_name(name))? - .map_err(|e| e.to_string()) - } - Sheets::Xls(x) => { - contained("merged-cell listing", || x.merge_cells_by_sheet_name(name))? - .map_err(|e| e.to_string()) - } - _ => continue, - }; - match regions { - Ok(dims) if !dims.is_empty() => { - out.insert(name.clone(), dims); - } - Ok(_) => {} - Err(e) => log::warn!("skipping unreadable merged-region list for {name:?}: {e}"), - } - } - Ok(out) +fn not_a_workbook() -> ConvertError { + ConvertError::malformed("not a readable workbook container") } -fn map_open_error(e: calamine::Error) -> ConvertError { - let text = e.to_string(); - if text.to_ascii_lowercase().contains("password") { - ConvertError::Encrypted - } else { - ConvertError::malformed(format!("unreadable workbook: {text}")) +enum Container { + Xml, + Bin, +} + +/// The package's main part: the root relationship names it wherever it +/// lives, so it decides ahead of the conventional locations, which a +/// package may also contain as leftovers. +fn main_part(pkg: &mut Package) -> Result, ConvertError> { + if let Some(target) = read_rels(pkg, "_rels/.rels")? + .first_of_type(rel_type::OFFICE_DOCUMENT) + .and_then(|rel| path::resolve("", &rel.target).ok()) + { + return Ok(Some(target.path)); } + Ok(["xl/workbook.xml", "xl/workbook.bin"] + .into_iter() + .find(|name| pkg.has_part(name)) + .map(str::to_string)) } -fn format_data(data: &Data) -> String { - match data { - Data::Empty => String::new(), - // Untrimmed: leading/trailing whitespace in a cell is source content. - Data::String(s) => clean_text(s), - Data::Float(f) => format_float(*f), - Data::Int(i) => i.to_string(), - Data::Bool(b) => if *b { "TRUE" } else { "FALSE" }.to_string(), - Data::Error(e) => format!("#{e:?}"), - Data::DateTime(dt) if dt.is_duration() => format_duration_days(dt.as_f64()), - // A serial below one whole day carries no date: it is a time of day. - Data::DateTime(dt) if dt.as_f64().abs() < 1.0 => format_time_of_day(dt.as_f64()), - Data::DateTime(dt) => match dt.as_datetime() { - Some(d) => { - let s = d.to_string(); - // Sub-second digits are noise from the serial's float. - let s = s.split('.').next().unwrap_or(&s); - s.strip_suffix(" 00:00:00").unwrap_or(s).to_string() - } - None => format_float(dt.as_f64()), - }, - Data::DateTimeIso(s) | Data::DurationIso(s) => s.clone(), +/// The container a main part belongs to, from its own bytes rather than its +/// name: a SpreadsheetML workbook is XML, an xlsb one is a record stream. +fn classify(pkg: &mut Package, part: &str) -> Result, ConvertError> { + let Some(bytes) = pkg.part(part)? else { + return Ok(None); + }; + let body = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(&bytes); + // A UTF-16 part opens on its byte order mark rather than the tag. + let is_xml = matches!(body.iter().find(|b| !b.is_ascii_whitespace()), Some(b'<' | 0xFF | 0xFE)); + if is_xml { + return Ok(Some(Container::Xml)); } + Ok((!body.is_empty()).then_some(Container::Bin)) } /// Float formatting at the 15 significant decimal digits a spreadsheet @@ -206,74 +108,37 @@ fn format_duration_days(days: f64) -> String { format!("{sign}{h}:{m:02}:{s:02}") } +/// Decode an RK value, the packed number both binary containers use. Bit 0 +/// asks for a hundredth of the result; bit 1 selects an integer over the +/// high 30 bits of a double. +pub(super) fn rk_number(rk: u32) -> f64 { + let value = if rk & 0x02 != 0 { + f64::from((rk as i32) >> 2) + } else { + f64::from_bits(u64::from(rk & 0xFFFF_FFFC) << 32) + }; + if rk & 0x01 != 0 { value / 100.0 } else { value } +} + +/// The literal an error code displays as, shared by both binary containers. +pub(super) fn error_literal(code: u8) -> Option<&'static str> { + Some(match code { + 0x00 => "#NULL!", + 0x07 => "#DIV/0!", + 0x0F => "#VALUE!", + 0x17 => "#REF!", + 0x1D => "#NAME?", + 0x24 => "#NUM!", + 0x2A => "#N/A", + 0x2B => "#GETTING_DATA", + _ => return None, + }) +} + #[cfg(test)] mod tests { use super::*; - use std::io::Write; - - /// Minimal xlsx with a used range at D11:E12 and the given merged region. - fn xlsx_with_merge(merge_ref: &str) -> Vec { - let sheet = format!( - r#"xyzw"# - ); - let parts: &[(&str, &str)] = &[ - ( - "[Content_Types].xml", - r#""#, - ), - ( - "_rels/.rels", - r#""#, - ), - ( - "xl/workbook.xml", - r#""#, - ), - ( - "xl/_rels/workbook.xml.rels", - r#""#, - ), - ]; - let mut w = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); - for (name, body) in parts { - w.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); - w.write_all(body.as_bytes()).unwrap(); - } - w.start_file("xl/worksheets/sheet1.xml", zip::write::SimpleFileOptions::default()).unwrap(); - w.write_all(sheet.as_bytes()).unwrap(); - w.finish().unwrap().into_inner() - } - - fn covered_count(doc: &Document) -> usize { - let Some(Block::Table(t)) = doc.blocks.first() else { - panic!("expected a table, got {:?}", doc.blocks.first()); - }; - t.grid - .iter() - .flatten() - .filter(|s| matches!(s, crate::model::CellSlot::Covered { .. })) - .count() - } - - #[test] - fn merge_inside_the_used_range_covers_cells() { - // Harness sanity: an in-range merge must actually load and apply. - let doc = parse(&xlsx_with_merge("D11:E11")).unwrap(); - assert_eq!(covered_count(&doc), 1); - } - - #[test] - fn merge_outside_the_used_columns_is_ignored() { - // M6: the merge overlaps the used rows but not the used columns; the - // old relative saturation mapped it onto (0,0) and covered D12. - let doc = parse(&xlsx_with_merge("A1:B12")).unwrap(); - assert_eq!(covered_count(&doc), 0, "out-of-range merge must not cover cells"); - } - - #[test] - fn string_cells_are_not_trimmed() { - assert_eq!(format_data(&Data::String(" padded ".into())), " padded "); - } + use std::io::Cursor; #[test] fn tiny_floats_survive() { @@ -307,4 +172,40 @@ mod tests { assert_eq!(format_duration_days(days), "26:30:15"); assert_eq!(format_duration_days(-0.5), "-12:00:00"); } + + #[test] + fn an_encrypted_ooxml_package_is_not_read_as_biff() { + // An encrypted workbook is an OLE container carrying no BIFF + // workbook stream, so the container check alone would send it to + // the wrong reader. + let mut ole = cfb::CompoundFile::create(Cursor::new(Vec::new())).unwrap(); + ole.create_stream("EncryptionInfo").unwrap(); + ole.create_stream("EncryptedPackage").unwrap(); + let bytes = ole.into_inner().into_inner(); + + assert!(matches!(parse(&bytes), Err(ConvertError::Encrypted))); + } + + #[test] + fn a_package_that_is_not_a_workbook_does_not_convert_as_one() { + use std::io::Write as _; + + // Resolving the root relationship reaches any OOXML main part, so + // without a workbook check a document would convert to nothing. + const REL: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + let rels = format!( + r#""# + ); + let doc = r#""#; + + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + for (name, body) in [("_rels/.rels", rels.as_str()), ("word/document.xml", doc)] { + zip.start_file(name, opts).unwrap(); + zip.write_all(body.as_bytes()).unwrap(); + } + let bytes = zip.finish().unwrap().into_inner(); + + assert!(matches!(parse(&bytes), Err(ConvertError::Malformed { .. }))); + } } diff --git a/src/formats/sheet/numfmt.rs b/src/formats/sheet/numfmt.rs new file mode 100644 index 00000000..c4ddda1d --- /dev/null +++ b/src/formats/sheet/numfmt.rs @@ -0,0 +1,1328 @@ +//! SpreadsheetML number format codes (ISO/IEC 29500-1 §18.8.30/31). +//! +//! A code parses into up to four `;`-separated sections. Numeric sections +//! render the value here; date/time sections are only classified, because +//! the sheet reader deliberately keeps its ISO-like date output (`mm/dd` +//! versus `dd/mm` is ambiguous for a downstream reader, ISO is not). Any +//! construct outside the implemented grammar makes [`NumberFormat::parse`] +//! return `None` and the caller falls back to General: approximating a +//! format silently would be worse than not applying it. + +use std::cmp::Ordering; + +/// Implied format codes for built-in numFmtIds. Ids 5-8 are absent +/// deliberately: the standard leaves them to the file's own formatCode, so +/// an unresolved reference falls back to General rather than a guessed +/// currency format. Ids 27-36 and 50-81 are locale-specific (zh, ja, ko, th) +/// and unresolvable without a locale. +pub(super) fn builtin_code(id: u32) -> Option<&'static str> { + Some(match id { + 1 => "0", + 2 => "0.00", + 3 => "#,##0", + 4 => "#,##0.00", + 9 => "0%", + 10 => "0.00%", + 11 => "0.00E+00", + 12 => "# ?/?", + 13 => "# ??/??", + 14 => "mm-dd-yy", + 15 => "d-mmm-yy", + 16 => "d-mmm", + 17 => "mmm-yy", + 18 => "h:mm AM/PM", + 19 => "h:mm:ss AM/PM", + 20 => "h:mm", + 21 => "h:mm:ss", + 22 => "m/d/yy h:mm", + 37 => "#,##0 ;(#,##0)", + 38 => "#,##0 ;[Red](#,##0)", + 39 => "#,##0.00;(#,##0.00)", + 40 => "#,##0.00;[Red](#,##0.00)", + 45 => "mm:ss", + 46 => "[h]:mm:ss", + 47 => "mmss.0", + 48 => "##0.0E+0", + 49 => "@", + _ => return None, + }) +} + +/// What a resolved format asks the caller to do with a numeric value. +#[derive(Debug, PartialEq)] +pub(super) enum Rendered<'a> { + /// Render this value the way an unformatted cell renders. + General { value: f64, prefix: &'a str, suffix: &'a str }, + /// The section is a date/time format: render the serial as the parts it + /// asks for. + DateTime(DateParts), + /// The formatted text, ready to emit. + Text(String), +} + +/// Which of a date/time format's parts it asks to see. A code naming only +/// months and days must not gain a time, and one naming only hours must not +/// gain a date. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub(super) struct DateParts { + pub(super) date: bool, + pub(super) time: bool, + /// `[h]`, `[m]` or `[s]`: a span rather than a point in time. + pub(super) elapsed: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum Region { + Int, + Frac, + Exp, + Num, + Den, +} + +#[derive(Debug, Clone, PartialEq)] +enum Tok { + Digit { + place: char, + region: Region, + }, + Decimal, + Percent, + Literal(String), + /// Unquoted digits, valid only as a fixed fraction denominator. + BareDigits(String), + /// A `,` pending resolution into grouping, scaling, or a literal. + Comma, + /// `E+` / `E-`; `plus` keeps the sign on non-negative exponents. + Exp { + plus: bool, + }, + /// Fraction bar between numerator and denominator placeholders. + Slash, + /// `@`, the text placeholder. + At, + /// `_x`: skip the width of one character (one space here). + Skip, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +enum Op { + Lt, + Le, + Gt, + Ge, + Eq, + Ne, +} + +#[derive(Debug, Clone, Copy)] +struct Cond { + op: Op, + operand: f64, +} + +impl Cond { + fn matches(&self, v: f64) -> bool { + match self.op { + Op::Lt => v < self.operand, + Op::Le => v <= self.operand, + Op::Gt => v > self.operand, + Op::Ge => v >= self.operand, + Op::Eq => v == self.operand, + Op::Ne => v != self.operand, + } + } +} + +#[derive(Debug)] +struct NumSpec { + toks: Vec, + grouping: bool, + /// Trailing commas: each divides the value by 1000. + scale: u32, + /// Each `%` multiplies the value by 100. + percents: u32, + int_places: usize, + frac_places: usize, + exp: bool, + num_places: usize, + den_places: usize, + /// Fraction denominator written as literal digits. + fixed_den: Option, +} + +#[derive(Debug)] +enum Body { + /// `General` with the literals that decorate it, which render around the + /// value the way an unformatted cell would show it. + General { + prefix: String, + suffix: String, + }, + DateTime(DateParts), + Number(NumSpec), + Text(Vec), +} + +#[derive(Debug)] +struct Section { + condition: Option, + body: Body, +} + +/// A parsed format code. +#[derive(Debug)] +pub(super) struct NumberFormat { + sections: Vec
, +} + +impl NumberFormat { + pub(super) fn parse(code: &str) -> Option { + // An empty code is not the `;;;` idiom for hiding a value, it is a + // format that says nothing, so the caller falls back to General. + if code.is_empty() { + return None; + } + let parts = split_sections(code)?; + if parts.is_empty() || parts.len() > 4 { + return None; + } + let sections: Vec
= + parts.iter().map(|p| parse_section(p)).collect::>()?; + if sections.iter().filter(|s| s.condition.is_some()).count() > 2 { + return None; + } + // A text section is only valid in last position; the fourth section + // is the text position, so nothing else may sit there. + let last = sections.len() - 1; + for (i, s) in sections.iter().enumerate() { + let is_text = matches!(s.body, Body::Text(_)); + if is_text && (i != last || s.condition.is_some()) { + return None; + } + if sections.len() == 4 && i == 3 && !is_text && !is_empty_body(&s.body) { + return None; + } + } + Some(NumberFormat { sections }) + } + + fn numeric_sections(&self) -> &[Section] { + match self.sections.last() { + Some(s) if matches!(s.body, Body::Text(_)) => &self.sections[..self.sections.len() - 1], + _ if self.sections.len() == 4 => &self.sections[..3], + _ => &self.sections, + } + } + + pub(super) fn format_number(&self, v: f64) -> Rendered<'_> { + if !v.is_finite() { + return Rendered::General { value: v, prefix: "", suffix: "" }; + } + let Some((section, value, auto_minus)) = select(self.numeric_sections(), v) else { + return Rendered::General { value: v, prefix: "", suffix: "" }; + }; + match §ion.body { + Body::General { prefix, suffix } => Rendered::General { value, prefix, suffix }, + Body::DateTime(parts) => Rendered::DateTime(*parts), + Body::Number(spec) => { + match render_number(spec, value.abs(), auto_minus && value < 0.0) { + Some(s) => Rendered::Text(s), + None => Rendered::General { value: v, prefix: "", suffix: "" }, + } + } + Body::Text(_) => Rendered::General { value: v, prefix: "", suffix: "" }, + } + } + + /// Apply the text section to a text value; `None` means the format has + /// no section for text, so the value stays as it is. + pub(super) fn format_text(&self, text: &str) -> Option { + let section = match self.sections.last() { + Some(s) if matches!(s.body, Body::Text(_)) => s, + _ if self.sections.len() == 4 => &self.sections[3], + _ => return None, + }; + match §ion.body { + Body::Text(toks) => { + let mut out = String::new(); + for tok in toks { + match tok { + Tok::Literal(s) => out.push_str(s), + Tok::Skip => out.push(' '), + Tok::At => out.push_str(text), + _ => {} + } + } + Some(out) + } + // An empty fourth section hides text. + _ => Some(String::new()), + } + } +} + +fn is_empty_body(body: &Body) -> bool { + matches!(body, Body::Number(spec) if spec.toks.is_empty()) +} + +/// Pick the section for a value. Returns the section, the value to render +/// (magnitude only for the positional negative section), and whether a +/// leading minus must be emitted for negative values. +fn select(sections: &[Section], v: f64) -> Option<(&Section, f64, bool)> { + if sections.is_empty() { + return None; + } + if sections.iter().any(|s| s.condition.is_some()) { + for s in sections { + match s.condition { + Some(c) if c.matches(v) => return Some((s, v, true)), + Some(_) => {} + None => return Some((s, v, true)), + } + } + return None; + } + let idx = match sections.len() { + 1 => 0, + 2 if v >= 0.0 => 0, + 2 => 1, + _ if v > 0.0 => 0, + _ if v < 0.0 => 1, + _ => 2, + }; + // The positional negative section renders the magnitude; its code + // supplies the sign (parens, a literal minus). + let neg_positional = idx == 1; + Some((§ions[idx], if neg_positional { v.abs() } else { v }, !neg_positional)) +} + +/// Split a code on `;` outside quotes, brackets, and escapes. +fn split_sections(code: &str) -> Option> { + let mut parts = vec![String::new()]; + let mut chars = code.chars(); + while let Some(c) = chars.next() { + match c { + ';' => parts.push(String::new()), + '"' | '[' => { + let close = if c == '"' { '"' } else { ']' }; + let part = parts.last_mut().unwrap(); + part.push(c); + loop { + let c = chars.next()?; + part.push(c); + if c == close { + break; + } + } + } + '\\' | '_' | '*' => { + let next = chars.next()?; + let part = parts.last_mut().unwrap(); + part.push(c); + part.push(next); + } + c => parts.last_mut().unwrap().push(c), + } + } + Some(parts) +} + +const COLORS: &[&str] = &["black", "blue", "cyan", "green", "magenta", "red", "white", "yellow"]; + +/// The text literal tokens render as, used to carry the decoration around a +/// `General` body. +fn decoration(toks: &[Tok]) -> String { + toks.iter() + .map(|t| match t { + Tok::Literal(s) => s.as_str(), + _ => " ", + }) + .collect() +} + +/// Which parts a date/time section asks for. `m` is minutes when an hour run +/// precedes it or a seconds run follows it, and months otherwise. +fn date_parts(runs: &[char], elapsed: bool) -> DateParts { + let mut parts = DateParts { elapsed, ..DateParts::default() }; + for (i, &run) in runs.iter().enumerate() { + match run { + 'y' | 'd' => parts.date = true, + 'h' | 's' | 'a' => parts.time = true, + 'm' => { + if runs[..i].last() == Some(&'h') || runs.get(i + 1) == Some(&'s') { + parts.time = true; + } else { + parts.date = true; + } + } + _ => {} + } + } + // An elapsed span is a duration whatever else the section names (`[m]` + // is elapsed minutes, never months), and a section with no run at all + // came from a bracket alone: both are a time and only a time. + if elapsed || (!parts.date && !parts.time) { + parts.date = false; + parts.time = true; + } + parts +} + +fn push_literal(raw: &mut Vec, c: char) { + match raw.last_mut() { + Some(Tok::Literal(s)) => s.push(c), + _ => raw.push(Tok::Literal(c.to_string())), + } +} + +fn parse_section(s: &str) -> Option
{ + let chars: Vec = s.chars().collect(); + let mut i = 0; + let mut raw: Vec = Vec::new(); + let mut general_at = 0usize; + let mut condition: Option = None; + let mut has_date = false; + let mut elapsed = false; + // The date/time runs in order, one letter each, so `m` can be read as + // months or minutes from what sits beside it. + let mut runs: Vec = Vec::new(); + let mut has_general = false; + while i < chars.len() { + let c = chars[i]; + match c { + '[' => { + let end = chars[i..].iter().position(|&c| c == ']')? + i; + let inner: String = chars[i + 1..end].iter().collect(); + i = end + 1; + bracket(&inner, &mut raw, &mut condition, &mut has_date, &mut elapsed, &mut runs)?; + } + '"' => { + let end = chars[i + 1..].iter().position(|&c| c == '"')? + i + 1; + for &c in &chars[i + 1..end] { + push_literal(&mut raw, c); + } + i = end + 1; + } + '\\' => { + push_literal(&mut raw, *chars.get(i + 1)?); + i += 2; + } + '_' => { + chars.get(i + 1)?; + raw.push(Tok::Skip); + i += 2; + } + '*' => { + // Repeat-to-fill: no column width exists here, emit nothing. + chars.get(i + 1)?; + i += 2; + } + '0' | '#' | '?' => { + raw.push(Tok::Digit { place: c, region: Region::Int }); + i += 1; + } + '.' => { + raw.push(Tok::Decimal); + i += 1; + } + ',' => { + raw.push(Tok::Comma); + i += 1; + } + '%' => { + raw.push(Tok::Percent); + i += 1; + } + '@' => { + raw.push(Tok::At); + i += 1; + } + 'E' | 'e' if matches!(chars.get(i + 1), Some('+') | Some('-')) => { + raw.push(Tok::Exp { plus: chars[i + 1] == '+' }); + i += 2; + } + 'y' | 'Y' | 'd' | 'D' | 'h' | 'H' | 's' | 'S' | 'm' | 'M' => { + has_date = true; + runs.push(c.to_ascii_lowercase()); + while i < chars.len() && chars[i].eq_ignore_ascii_case(&c) { + i += 1; + } + } + 'g' | 'G' => { + let word: String = chars[i..chars.len().min(i + 7)].iter().collect(); + if !word.eq_ignore_ascii_case("general") { + return None; + } + has_general = true; + // Adjacent literals merge, so the split has to be recorded + // against the rendered text rather than a token index. + general_at = decoration(&raw).chars().count(); + i += 7; + } + 'a' | 'A' => { + // Matched over chars: the tokens are ASCII, but a multibyte + // char can follow them, and byte slicing would split it. + let len = ["AM/PM", "A/P"] + .iter() + .find(|t| { + chars.len() - i >= t.len() + && chars[i..i + t.len()] + .iter() + .zip(t.chars()) + .all(|(c, want)| c.eq_ignore_ascii_case(&want)) + }) + .map(|t| t.len())?; + has_date = true; + runs.push('a'); + i += len; + } + '1'..='9' => { + let end = chars[i..] + .iter() + .position(|c| !c.is_ascii_digit()) + .map_or(chars.len(), |p| p + i); + raw.push(Tok::BareDigits(chars[i..end].iter().collect())); + i = end; + } + '$' | '-' | '+' | '(' | ')' | ':' | ' ' => { + push_literal(&mut raw, c); + i += 1; + } + '/' => { + raw.push(Tok::Slash); + i += 1; + } + _ => return None, + } + } + let body = if has_general { + if raw.iter().any(|t| !matches!(t, Tok::Literal(_) | Tok::Skip)) { + return None; + } + let text = decoration(&raw); + let split = text.char_indices().nth(general_at).map_or(text.len(), |(b, _)| b); + Body::General { prefix: text[..split].to_string(), suffix: text[split..].to_string() } + } else if has_date { + if raw.iter().any(|t| matches!(t, Tok::At | Tok::Exp { .. } | Tok::BareDigits(_))) { + return None; + } + Body::DateTime(date_parts(&runs, elapsed)) + } else if raw.iter().any(|t| matches!(t, Tok::At)) { + if raw.iter().any(|t| { + matches!( + t, + Tok::Digit { .. } + | Tok::Decimal + | Tok::Exp { .. } + | Tok::Slash + | Tok::BareDigits(_) + ) + }) { + return None; + } + let toks = raw + .into_iter() + .map(|t| match t { + Tok::Percent => Tok::Literal("%".to_string()), + Tok::Comma => Tok::Literal(",".to_string()), + t => t, + }) + .collect(); + Body::Text(toks) + } else { + Body::Number(resolve_number(raw)?) + }; + Some(Section { condition, body }) +} + +fn bracket( + inner: &str, + raw: &mut Vec, + condition: &mut Option, + has_date: &mut bool, + elapsed: &mut bool, + runs: &mut Vec, +) -> Option<()> { + match inner.chars().next()? { + '<' | '>' | '=' => { + if condition.is_some() { + return None; + } + let (op, rest) = if let Some(r) = inner.strip_prefix(">=") { + (Op::Ge, r) + } else if let Some(r) = inner.strip_prefix("<=") { + (Op::Le, r) + } else if let Some(r) = inner.strip_prefix("<>") { + (Op::Ne, r) + } else if let Some(r) = inner.strip_prefix('>') { + (Op::Gt, r) + } else if let Some(r) = inner.strip_prefix('<') { + (Op::Lt, r) + } else { + (Op::Eq, inner.strip_prefix('=')?) + }; + *condition = Some(Cond { op, operand: rest.trim().parse().ok()? }); + } + '$' => { + // `[$sym-lcid]`: the currency string emits literally, the + // locale id affects nothing rendered here. + let sym = inner[1..].split('-').next().unwrap_or(""); + if !sym.is_empty() { + raw.push(Tok::Literal(sym.to_string())); + } + } + c @ ('h' | 'H' | 'm' | 'M' | 's' | 'S') + if inner.chars().all(|x| x.eq_ignore_ascii_case(&c)) => + { + *has_date = true; + *elapsed = true; + runs.push(c.to_ascii_lowercase()); + } + _ => { + let lower = inner.to_ascii_lowercase(); + let is_color = COLORS.contains(&lower.as_str()) + || lower + .strip_prefix("color") + .and_then(|n| n.trim().parse::().ok()) + .is_some_and(|n| (1..=56).contains(&n)); + if !is_color { + return None; + } + } + } + Some(()) +} + +/// Second pass over a numeric section: assign digit regions, resolve commas +/// into grouping or scaling, and recognize the fraction form. +fn resolve_number(raw: Vec) -> Option { + let mut toks: Vec = Vec::new(); + let mut region = Region::Int; + let mut spec = NumSpec { + toks: Vec::new(), + grouping: false, + scale: 0, + percents: 0, + int_places: 0, + frac_places: 0, + exp: false, + num_places: 0, + den_places: 0, + fixed_den: None, + }; + let mut iter = raw.into_iter().peekable(); + while let Some(tok) = iter.next() { + match tok { + Tok::Digit { place, .. } => { + if spec.fixed_den.is_some() && region == Region::Den { + return None; + } + toks.push(Tok::Digit { place, region }); + } + Tok::Decimal => { + if region != Region::Int { + return None; + } + region = Region::Frac; + toks.push(Tok::Decimal); + } + Tok::Exp { plus } => { + if spec.exp || matches!(region, Region::Num | Region::Den) { + return None; + } + spec.exp = true; + region = Region::Exp; + toks.push(Tok::Exp { plus }); + } + Tok::Slash => { + // A fraction bar needs a numerator run directly before it; + // otherwise the slash is a literal. + let run = toks + .iter() + .rev() + .take_while(|t| matches!(t, Tok::Digit { region: Region::Int, .. })) + .count(); + if run == 0 || region != Region::Int || spec.fixed_den.is_some() { + push_literal(&mut toks, '/'); + continue; + } + let at = toks.len() - run; + for t in &mut toks[at..] { + if let Tok::Digit { region, .. } = t { + *region = Region::Num; + } + } + toks.push(Tok::Slash); + region = Region::Den; + if let Some(Tok::BareDigits(_)) = iter.peek() { + let Some(Tok::BareDigits(d)) = iter.next() else { unreachable!() }; + spec.fixed_den = Some(d.parse().ok()?); + toks.push(Tok::Literal(d)); + } + } + Tok::Percent => { + spec.percents += 1; + toks.push(Tok::Percent); + } + // Unquoted digits anywhere but a fixed denominator are outside + // the grammar. + Tok::BareDigits(_) => return None, + other => toks.push(other), + } + } + // Commas between digit placeholders group; commas after the last digit + // placeholder scale by 1000 each; the rest are literal. + let digit_at: Vec = toks.iter().map(|t| matches!(t, Tok::Digit { .. })).collect(); + let first_digit = digit_at.iter().position(|&d| d); + let last_digit = digit_at.iter().rposition(|&d| d); + let mut out: Vec = Vec::new(); + for (i, tok) in toks.into_iter().enumerate() { + if tok != Tok::Comma { + out.push(tok); + continue; + } + match (first_digit, last_digit) { + (Some(_), Some(l)) if i > l => spec.scale += 1, + (Some(f), Some(l)) if i > f && i < l => spec.grouping = true, + _ => push_literal(&mut out, ','), + } + } + for t in &out { + if let Tok::Digit { region, .. } = t { + match region { + Region::Int => spec.int_places += 1, + Region::Frac => spec.frac_places += 1, + Region::Exp => {} + Region::Num => spec.num_places += 1, + Region::Den => spec.den_places += 1, + } + } + } + if spec.exp && !out.iter().any(|t| matches!(t, Tok::Digit { region: Region::Exp, .. })) { + return None; + } + if spec.num_places > 0 && spec.den_places == 0 && spec.fixed_den.is_none() { + return None; + } + spec.toks = out; + Some(spec) +} + +fn render_number(spec: &NumSpec, v_abs: f64, minus: bool) -> Option { + let mut v = v_abs; + for _ in 0..spec.percents { + v *= 100.0; + } + for _ in 0..spec.scale { + v /= 1000.0; + } + if !v.is_finite() { + return None; + } + let body = if spec.exp { + render_scientific(spec, v)? + } else if spec.num_places > 0 { + render_fraction(spec, v)? + } else { + let (int_digits, frac_digits) = split_digits(v, spec.frac_places)?; + emit(spec, &int_digits, &frac_digits, "", 0) + }; + Some(if minus { format!("-{body}") } else { body }) +} + +/// The rounded value's digits: the integer part (empty when zero, so `#` +/// can drop it) and exactly `dp` fractional digits. Rounding happens on the +/// 15-significant-digit decimal form, half away from zero, the way a +/// spreadsheet displays - binary arithmetic would round 5.255 at two +/// decimals to 5.25. +fn split_digits(v: f64, dp: usize) -> Option<(String, String)> { + if !v.is_finite() || v < 0.0 || dp > 512 { + return None; + } + if v == 0.0 { + return Some((String::new(), "0".repeat(dp))); + } + // `{:.14e}` is the value at 15 significant decimal digits: an integer D + // of up to 15 digits and an exponent, v = D * 10^(e-14). + let repr = format!("{v:.14e}"); + let (mantissa, e) = repr.split_once('e')?; + let e: i64 = e.parse().ok()?; + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + // Digits of round(v * 10^dp), by shifting or by decimal rounding. + let shift = e - 14 + dp as i64; + let scaled = if shift >= 0 { + format!("{digits}{}", "0".repeat(usize::try_from(shift).ok()?)) + } else { + let drop = usize::try_from(-shift).ok()?; + if drop > digits.len() { + String::new() + } else { + let (kept, rest) = digits.split_at(digits.len() - drop); + let mut kept: Vec = kept.bytes().collect(); + if rest.as_bytes().first().is_some_and(|&b| b >= b'5') { + let mut i = kept.len(); + loop { + if i == 0 { + kept.insert(0, b'1'); + break; + } + i -= 1; + if kept[i] == b'9' { + kept[i] = b'0'; + } else { + kept[i] += 1; + break; + } + } + } + String::from_utf8(kept).ok()? + } + }; + let scaled = scaled.trim_start_matches('0'); + let mut s = scaled.to_string(); + if s.len() < dp { + s = format!("{}{}", "0".repeat(dp - s.len()), s); + } + let (int, frac) = s.split_at(s.len() - dp); + Some((int.to_string(), frac.to_string())) +} + +fn places(toks: &[Tok], r: Region) -> Vec { + toks.iter() + .filter_map(|t| match t { + Tok::Digit { place, region } if *region == r => Some(*place), + _ => None, + }) + .collect() +} + +/// Distribute a digit string over placeholders: the first takes any excess +/// digits, and placeholders past the available digits pad per their kind +/// (`0` a zero, `?` a space, `#` nothing). +fn assign(digits: &str, places: &[char]) -> Vec { + let k = places.len(); + if k == 0 { + return Vec::new(); + } + let n = digits.len(); + let mut out = Vec::with_capacity(k); + if n > k { + out.push(digits[..n - k + 1].to_string()); + for c in digits[n - k + 1..].chars() { + out.push(c.to_string()); + } + } else { + for &p in &places[..k - n] { + out.push(match p { + '0' => "0".to_string(), + '?' => " ".to_string(), + _ => String::new(), + }); + } + for c in digits.chars() { + out.push(c.to_string()); + } + } + out +} + +/// Walk the token list emitting digits into placeholders. +fn emit( + spec: &NumSpec, + int_digits: &str, + frac_digits: &str, + exp_digits: &str, + exp_sign: i8, +) -> String { + let int_assigned = assign(int_digits, &places(&spec.toks, Region::Int)); + let exp_assigned = assign(exp_digits, &places(&spec.toks, Region::Exp)); + let is_digit = |c: &char| c.is_ascii_digit(); + let total_int: usize = int_assigned.iter().map(|s| s.chars().filter(is_digit).count()).sum(); + // Fractional digits kept: everything up to the rightmost `0` placeholder + // always shows; beyond that, trailing zeros drop from `#` and pad `?`. + let frac_places = places(&spec.toks, Region::Frac); + let min_frac = frac_places.iter().rposition(|&p| p == '0').map_or(0, |i| i + 1); + let keep_frac = frac_digits.trim_end_matches('0').len().max(min_frac); + + // The point hides only when its placeholders all produced nothing; a + // point with no placeholder at all (`0.`) is written as it stands. + let show_decimal = keep_frac > 0 || frac_places.contains(&'?') || frac_places.is_empty(); + + let mut out = String::new(); + let (mut int_i, mut frac_i, mut exp_i) = (0usize, 0usize, 0usize); + let mut int_remaining = total_int; + for tok in &spec.toks { + match tok { + Tok::Digit { region: Region::Int, .. } => { + for c in int_assigned[int_i].chars() { + out.push(c); + if c.is_ascii_digit() { + int_remaining -= 1; + if spec.grouping && int_remaining > 0 && int_remaining.is_multiple_of(3) { + out.push(','); + } + } + } + int_i += 1; + } + Tok::Digit { place, region: Region::Frac } => { + if frac_i < keep_frac { + out.push(frac_digits.as_bytes()[frac_i] as char); + } else if *place == '?' { + out.push(' '); + } + frac_i += 1; + } + Tok::Digit { region: Region::Exp, .. } => { + out.push_str(&exp_assigned[exp_i]); + exp_i += 1; + } + Tok::Digit { .. } => {} + Tok::Decimal => { + if show_decimal { + out.push('.'); + } + } + Tok::Percent => out.push('%'), + Tok::Literal(s) => out.push_str(s), + Tok::Exp { plus } => { + out.push('E'); + if exp_sign < 0 { + out.push('-'); + } else if *plus { + out.push('+'); + } + } + Tok::Skip => out.push(' '), + Tok::Slash | Tok::At | Tok::Comma | Tok::BareDigits(_) => {} + } + } + out +} + +fn render_scientific(spec: &NumSpec, v: f64) -> Option { + let n_int = spec.int_places.max(1) as i64; + let (int_digits, frac_digits, exp10) = if v == 0.0 { + let (i, f) = split_digits(0.0, spec.frac_places)?; + (i, f, 0i64) + } else { + // The decimal exponent comes from the shortest round-trip + // formatting; the float log10 misplaces boundaries like 1000. + let repr = format!("{v:e}"); + let mut e: i64 = repr.split('e').nth(1)?.parse().ok()?; + // The exponent stays a multiple of the integer placeholder count + // (engineering notation for `##0.0E+0`). + e = e.div_euclid(n_int) * n_int; + let m = v / 10f64.powi(i32::try_from(e).ok()?); + let (mut i, mut f) = split_digits(m, spec.frac_places)?; + // Rounding at the display precision can carry into a new digit + // (9.99 -> 10.0): renormalize. + if i.len() > usize::try_from(n_int).ok()? { + e += n_int; + let m = v / 10f64.powi(i32::try_from(e).ok()?); + (i, f) = split_digits(m, spec.frac_places)?; + } + (i, f, e) + }; + let bare = exp10.unsigned_abs().to_string(); + let pad = places(&spec.toks, Region::Exp).len().saturating_sub(bare.len()); + let exp_digits = format!("{}{}", "0".repeat(pad), bare); + Some(emit(spec, &int_digits, &frac_digits, &exp_digits, if exp10 < 0 { -1 } else { 1 })) +} + +fn render_fraction(spec: &NumSpec, v: f64) -> Option { + if v >= 1e15 { + return None; + } + let has_int = spec.int_places > 0; + let (mut whole, target) = if has_int { (v.trunc(), v.fract()) } else { (0.0, v) }; + let (mut num, den) = best_fraction(target, spec.fixed_den, spec.den_places)?; + if has_int && num == den && den > 0 { + whole += 1.0; + num = 0; + } + let int_digits = if whole == 0.0 { + // A zero integer part still shows when the whole value is zero. + if num == 0 { "0".to_string() } else { String::new() } + } else { + split_digits(whole, 0)?.0 + }; + // A zero numerator blanks the fraction: "5", not "5 0/1". + let hide = has_int && num == 0; + let mut int_assigned = assign(&int_digits, &places(&spec.toks, Region::Int)).into_iter(); + let mut num_assigned = assign(&num.to_string(), &places(&spec.toks, Region::Num)).into_iter(); + let mut den_assigned = assign(&den.to_string(), &places(&spec.toks, Region::Den)).into_iter(); + let mut out = String::new(); + for tok in &spec.toks { + match tok { + Tok::Digit { region: Region::Int, .. } => { + out.push_str(&int_assigned.next().unwrap_or_default()); + } + Tok::Digit { region: Region::Num, .. } => { + if !hide { + out.push_str(&num_assigned.next().unwrap_or_default()); + } + } + Tok::Digit { region: Region::Den, .. } => { + if !hide { + out.push_str(&den_assigned.next().unwrap_or_default()); + } + } + Tok::Slash => { + if !hide { + out.push('/'); + } + } + Tok::Literal(s) => { + // The fixed denominator is stored as a literal; it hides + // with the rest of the fraction. + if !(hide && spec.fixed_den.is_some_and(|d| d.to_string() == *s)) { + out.push_str(s); + } + } + Tok::Percent => out.push('%'), + Tok::Skip => out.push(' '), + _ => {} + } + } + Some(out.trim_end().to_string()) +} + +fn best_fraction(x: f64, fixed: Option, den_places: usize) -> Option<(u64, u64)> { + if x < 0.0 || !x.is_finite() { + return None; + } + if let Some(d) = fixed { + let n = (x * d as f64).round(); + return (n < 1e18).then_some((n as u64, d)); + } + // Past 19 places the power of ten overflows; every u64 denominator is + // within such a bound, so it saturates rather than dropping the format. + let max_den = u32::try_from(den_places) + .ok() + .and_then(|p| 10u64.checked_pow(p)) + .map_or(u64::MAX, |p| p.saturating_sub(1)) + .max(1); + Some(closest_rational(x, max_den)) +} + +/// The closest rational to `x` with a denominator no larger than `max_den`. +/// +/// Walks the continued fraction of the exact ratio the float stores, in a +/// number of steps proportional to the digits of `max_den`. Repeatedly +/// inverting the float remainder instead would compound rounding until the +/// walk picks a non-closest fraction under large bounds. When the next term +/// would overshoot the bound, the only other candidate is the semiconvergent +/// with the largest term that fits, and the exact tail decides between them. +fn closest_rational(x: f64, max_den: u64) -> (u64, u64) { + let Some((mut p, mut q)) = dyadic(x) else { + return (0, 1); + }; + let (mut pn, mut pd) = (0u64, 1u64); + let (mut n, mut d) = (1u64, 0u64); + while q > 0 { + let a = p / q; + let rem = p % q; + let next = u64::try_from(a).ok().and_then(|a| { + Some((a.checked_mul(n)?.checked_add(pn)?, a.checked_mul(d)?.checked_add(pd)?)) + }); + let (nn, nd) = match next { + Some((nn, nd)) if nd <= max_den => (nn, nd), + // The first term alone is past every u64 numerator. + _ if d == 0 => return (0, 1), + _ => { + let k = (max_den - pd) / d; + let semi = + k.checked_mul(n).and_then(|v| v.checked_add(pn)).map(|sn| (sn, k * d + pd)); + // With the exact tail r = p/q, the semiconvergent is closer + // iff (r - k)d < kd + pd, i.e. rem*d < q*((2k - a)d + pd): + // certain for 2k > a, impossible for 2k < a (pd < d). + let better = match (u128::from(k) * 2).cmp(&a) { + Ordering::Greater => true, + Ordering::Less => false, + Ordering::Equal => mul_cmp(rem, d, q, pd) == Ordering::Less, + }; + return match (semi, better) { + (Some(semi), true) => semi, + _ => (n, d), + }; + } + }; + (pn, pd, n, d) = (n, d, nn, nd); + (p, q) = (q, rem); + } + if d == 0 { (0, 1) } else { (n, d) } +} + +/// A finite non-negative float as the ratio it stores exactly. `None` when +/// no exact ratio fits (under 2^-74 every u64-bounded fraction rounds to +/// zero; past u128 no bounded denominator distinguishes it from an +/// integer), which the walk treats as zero. +fn dyadic(x: f64) -> Option<(u128, u128)> { + let bits = x.to_bits(); + let exp = (bits >> 52) as i32; + let frac = bits & ((1u64 << 52) - 1); + let (m, e) = if exp == 0 { (frac, -1074) } else { (frac | 1 << 52, exp - 1075) }; + if m == 0 { + return Some((0, 1)); + } + if e >= 0 { + return (e <= 75).then(|| (u128::from(m) << e, 1)); + } + let shift = m.trailing_zeros().min(e.unsigned_abs()); + let k = e.unsigned_abs() - shift; + (k <= 127).then(|| (u128::from(m >> shift), 1u128 << k)) +} + +/// `a * b` against `c * e`, exact in 192 bits. +fn mul_cmp(a: u128, b: u64, c: u128, e: u64) -> Ordering { + let wide = |x: u128, y: u64| { + let lo = (x & u128::from(u64::MAX)) * u128::from(y); + ((x >> 64) * u128::from(y) + (lo >> 64), lo as u64) + }; + wide(a, b).cmp(&wide(c, e)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fmt(code: &str, v: f64) -> String { + match NumberFormat::parse(code).expect("code must parse").format_number(v) { + Rendered::Text(s) => s, + other => panic!("expected text for {code:?} on {v}, got {other:?}"), + } + } + + #[test] + fn percent_scales_by_hundred() { + assert_eq!(fmt("0.0%", 0.075), "7.5%"); + assert_eq!(fmt("0%", 0.155), "16%"); + assert_eq!(fmt("0.00%", -0.5), "-50.00%"); + } + + #[test] + fn thousands_grouping() { + assert_eq!(fmt("#,##0", 9876543.0), "9,876,543"); + assert_eq!(fmt("#,##0.00", 1234.5), "1,234.50"); + assert_eq!(fmt("#,##0", 0.0), "0"); + assert_eq!(fmt("#,##0", 999.0), "999"); + } + + #[test] + fn quoted_and_bracketed_currency_pass_through() { + assert_eq!(fmt("\"$\"#,##0.00", 1234.5), "$1,234.50"); + assert_eq!(fmt("[$$-409]#,##0.00", 1234.5), "$1,234.50"); + assert_eq!(fmt("#,##0.00\\ \"kr\"", 1234.5), "1,234.50 kr"); + } + + #[test] + fn digit_placeholders_pad_per_kind() { + assert_eq!(fmt("00000", 42.0), "00042"); + assert_eq!(fmt("#", 0.0), ""); + assert_eq!(fmt("0.##", 5.0), "5"); + assert_eq!(fmt("0.??", 5.0), "5. "); + assert_eq!(fmt("0.0#", 5.25), "5.25"); + assert_eq!(fmt("0.00", 5.255), "5.26"); + assert_eq!(fmt("???", 42.0), " 42"); + } + + #[test] + fn a_decimal_point_with_no_placeholders_still_shows() { + assert_eq!(fmt("0.", 5.0), "5."); + assert_eq!(fmt("0.\"kg\"", 5.0), "5.kg"); + } + + #[test] + fn sections_map_by_sign() { + assert_eq!(fmt("0.00;(0.00)", -3.5), "(3.50)"); + assert_eq!(fmt("0.00;(0.00)", 3.5), "3.50"); + assert_eq!(fmt("0;-0;\"zero\"", 0.0), "zero"); + assert_eq!(fmt("0", -3.0), "-3"); + } + + #[test] + fn colors_are_discarded() { + assert_eq!(fmt("#,##0;[Red](#,##0)", -1234.0), "(1,234)"); + } + + #[test] + fn conditions_select_sections() { + assert_eq!(fmt("[>=100]0.0;0.00", 250.0), "250.0"); + assert_eq!(fmt("[>=100]0.0;0.00", 3.0), "3.00"); + } + + #[test] + fn scaling_commas_divide_by_thousand() { + assert_eq!(fmt("0.0,,", 12_345_678.0), "12.3"); + assert_eq!(fmt("#,##0,", 12_345_678.0), "12,346"); + } + + #[test] + fn scientific_notation() { + assert_eq!(fmt("0.00E+00", 12345.0), "1.23E+04"); + assert_eq!(fmt("0.00E+00", 0.0001234), "1.23E-04"); + assert_eq!(fmt("##0.0E+0", 0.0000123), "12.3E-6"); + assert_eq!(fmt("0.00E+00", 0.0), "0.00E+00"); + } + + #[test] + fn fractions_approximate() { + assert_eq!(fmt("# ?/?", 5.25), "5 1/4"); + assert_eq!(fmt("# ??/??", 2.675), "2 27/40"); + assert_eq!(fmt("# ?/?", 5.0), "5"); + assert_eq!(fmt("?/?", 0.5), "1/2"); + assert_eq!(fmt("# ?/8", 5.25), "5 2/8"); + } + + #[test] + fn twenty_denominator_places_still_render_a_fraction() { + let code = format!("?/{}", "?".repeat(20)); + assert_eq!(fmt(&code, 0.5), format!("1/{}2", " ".repeat(19))); + } + + #[test] + fn skip_emits_space_and_fill_emits_nothing() { + assert_eq!(fmt("0.00_);(0.00)", 3.5), "3.50 "); + assert_eq!(fmt("$* 0.00", 3.5), "$3.50"); + } + + #[test] + fn escaped_and_quoted_text_is_literal() { + assert_eq!(fmt("0.0\\ \"m/s\"", 3.51), "3.5 m/s"); + // Quoted date letters must not turn the section into a date. + assert_eq!(fmt("0\"d\"", 3.0), "3d"); + } + + #[test] + fn literal_only_sections_hide_the_value() { + assert_eq!(fmt("\"yes\";\"yes\";\"no\"", 1.0), "yes"); + assert_eq!(fmt("\"yes\";\"yes\";\"no\"", 0.0), "no"); + } + + #[test] + fn date_sections_classify_without_rendering() { + let f = NumberFormat::parse("yyyy\\-mm\\-dd").unwrap(); + assert_eq!( + f.format_number(45000.0), + Rendered::DateTime(DateParts { date: true, time: false, elapsed: false }) + ); + let f = NumberFormat::parse("[hh]:mm:ss").unwrap(); + assert_eq!( + f.format_number(1.5), + Rendered::DateTime(DateParts { date: false, time: true, elapsed: true }) + ); + let f = NumberFormat::parse("h:mm AM/PM").unwrap(); + assert_eq!( + f.format_number(0.5), + Rendered::DateTime(DateParts { date: false, time: true, elapsed: false }) + ); + } + + #[test] + fn general_keeps_the_literals_around_it() { + // `General"kg"` shows the unformatted value with its unit, so the + // literals cannot be dropped on the way through. + let f = NumberFormat::parse(r#""~"General" kg""#).unwrap(); + assert_eq!( + f.format_number(1234.5), + Rendered::General { value: 1234.5, prefix: "~", suffix: " kg" } + ); + } + + #[test] + fn general_renders_generally() { + let f = NumberFormat::parse("General").unwrap(); + assert_eq!(f.format_number(3.5), Rendered::General { value: 3.5, prefix: "", suffix: "" }); + // The positional negative section receives the magnitude. + let f = NumberFormat::parse("General;General").unwrap(); + assert_eq!(f.format_number(-3.5), Rendered::General { value: 3.5, prefix: "", suffix: "" }); + } + + #[test] + fn unsupported_constructs_refuse_to_parse() { + assert!(NumberFormat::parse("[DBNum1]0").is_none()); + assert!(NumberFormat::parse("0.0.0").is_none()); + assert!(NumberFormat::parse("abc0").is_none()); + assert!(NumberFormat::parse("0;0;0;0;0").is_none()); + // Unquoted currency letters are outside the implemented grammar. + assert!(NumberFormat::parse("€0.00").is_none()); + } + + #[test] + fn text_section_applies_to_text_only() { + let f = NumberFormat::parse("0.00;(0.00);\"-\";\"* \"@\" *\"").unwrap(); + assert_eq!(f.format_text("hi"), Some("* hi *".to_string())); + let f = NumberFormat::parse("@").unwrap(); + assert_eq!(f.format_text("hi"), Some("hi".to_string())); + assert_eq!(f.format_number(3.5), Rendered::General { value: 3.5, prefix: "", suffix: "" }); + let f = NumberFormat::parse("0.00").unwrap(); + assert_eq!(f.format_text("hi"), None); + } + + #[test] + fn fifteen_digit_rounding_applies_before_formatting() { + // 0.075 is stored just under 0.075; the percent path must still + // show 7.5, not 7.4. + assert_eq!(fmt("0.0%", 0.075), "7.5%"); + assert_eq!(fmt("0", 2.5), "3"); + } + + #[test] + fn a_date_section_names_only_the_parts_it_asks_for() { + let parts = |code: &str| match NumberFormat::parse(code).unwrap().format_number(45000.5) { + Rendered::DateTime(p) => (p.date, p.time, p.elapsed), + other => panic!("expected a date/time section, got {other:?}"), + }; + assert_eq!(parts("yyyy-mm-dd"), (true, false, false)); + assert_eq!(parts("d mmm yyyy"), (true, false, false)); + assert_eq!(parts("h:mm"), (false, true, false)); + assert_eq!(parts("yyyy-mm-dd hh:mm:ss"), (true, true, false)); + assert_eq!(parts("[h]:mm:ss"), (false, true, true)); + // A lone elapsed `m` is minutes with no neighbour to say so. + assert_eq!(parts("[m]"), (false, true, true)); + // `m` is minutes beside an hour or a second, and months otherwise. + assert_eq!(parts("mm:ss"), (false, true, false)); + assert_eq!(parts("mm/dd/yyyy"), (true, false, false)); + } + + #[test] + fn the_closest_rational_matches_an_exhaustive_search() { + // The continued fraction walk has to reach the same answer a scan of + // every denominator would, semiconvergents included. + let brute = |x: f64, max_den: u64| { + let mut best = (0u64, 1u64); + let mut err = f64::INFINITY; + for d in 1..=max_den { + let n = (x * d as f64).round(); + let e = (x - n / d as f64).abs(); + if e < err - 1e-15 { + err = e; + best = (n as u64, d); + } + } + best + }; + let mut seed = 12_345u64; + for _ in 0..2_000 { + seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1); + let x = (seed >> 11) as f64 / (1u64 << 53) as f64 * 10.0; + for places in 1..=4 { + let max_den = 10u64.pow(places) - 1; + let (n, d) = closest_rational(x, max_den); + let (bn, bd) = brute(x, max_den); + let got = (x - n as f64 / d as f64).abs(); + let want = (x - bn as f64 / bd as f64).abs(); + assert!(got <= want + 1e-12, "x={x} max_den={max_den}: {n}/{d} vs {bn}/{bd}"); + } + } + } + + #[test] + fn the_walk_stays_exact_past_float_precision() { + // 1/3 as a float is the dyadic 6004799503160661/2^54, which a large + // bound represents exactly; a float walk stops at 1/3 instead. + assert_eq!(closest_rational(1.0 / 3.0, u64::MAX), (6004799503160661, 18014398509481984)); + } +} diff --git a/src/formats/sheet/xls.rs b/src/formats/sheet/xls.rs new file mode 100644 index 00000000..450cb86b --- /dev/null +++ b/src/formats/sheet/xls.rs @@ -0,0 +1,1081 @@ +//! In-house legacy Excel reader (.xls): OLE2 compound file holding a BIFF +//! record stream ([MS-XLS]). BIFF8 is the target; BIFF5/BIFF7 streams +//! degrade to their byte-string record layouts instead of erroring. Cell +//! values resolve their number format through the same engine and grid +//! assembly as the SpreadsheetML reader, so a workbook saved as .xls and as +//! .xlsx converts identically. + +use super::xlsx::{ + CellFormat, SheetContent, build_table, format_as_text, render_number, resolve_format, +}; +use super::{error_literal, rk_number}; +use crate::error::ConvertError; +use crate::model::{Block, Document, Inline}; +use crate::package::limits; +use crate::shared::binary::{get_u16, get_u32, read_ole_stream}; +use crate::shared::text::clean_text; +use std::collections::HashMap; +use std::io::Cursor; + +// Record types ([MS-XLS] 2.3.2), BIFF5-BIFF8 numbering. +const BOF: u16 = 0x0809; +const EOF_REC: u16 = 0x000A; +const FILEPASS: u16 = 0x002F; +const CODEPAGE: u16 = 0x0042; +const DATEMODE: u16 = 0x0022; +const BOUNDSHEET: u16 = 0x0085; +const SST: u16 = 0x00FC; +const CONTINUE: u16 = 0x003C; +const FORMAT: u16 = 0x041E; +const XF: u16 = 0x00E0; +const ROW: u16 = 0x0208; +const COLINFO: u16 = 0x007D; +const MERGEDCELLS: u16 = 0x00E5; +const LABELSST: u16 = 0x00FD; +const LABEL: u16 = 0x0204; +const RSTRING: u16 = 0x00D6; +const NUMBER: u16 = 0x0203; +const RK: u16 = 0x027E; +const MULRK: u16 = 0x00BD; +const BOOLERR: u16 = 0x0205; +const FORMULA: u16 = 0x0006; +const STRING: u16 = 0x0207; + +/// BOF `dt` value for a worksheet (or dialog sheet) substream. +const WORKSHEET_SUBSTREAM: u16 = 0x0010; + +/// The BIFF8 grid is 256 columns; a larger column index is not a real cell. +const MAX_COLS: u32 = 256; + +pub(super) fn parse(bytes: &[u8]) -> Result { + let mut ole = cfb::CompoundFile::open(Cursor::new(bytes)) + .map_err(|e| ConvertError::malformed(format!("not an OLE2 compound file: {e}")))?; + let data = workbook_stream(&mut ole)?; + let mut records = 0u64; + let globals = read_globals(&data, &mut records)?; + + let visible: Vec<&BoundSheet> = globals.sheets.iter().filter(|s| s.visible).collect(); + let multi_sheet = visible.len() > 1; + let mut doc = Document::default(); + let mut failed = 0usize; + // One budget for the workbook, so sheets cannot multiply the cap. + let mut slots = 0u64; + for sheet in &visible { + let Some(content) = read_sheet(&data, &globals, sheet.offset, &mut records)? else { + log::warn!("skipping unreadable sheet {:?}", sheet.name); + failed += 1; + continue; + }; + let Some(table) = build_table(content, &mut slots)? else { + continue; + }; + if multi_sheet { + doc.blocks.push(Block::heading(2, vec![Inline::plain(sheet.name.clone())])); + } + doc.blocks.push(Block::Table(table)); + } + if !visible.is_empty() && failed == visible.len() { + return Err(ConvertError::malformed("no sheet in the workbook could be read")); + } + Ok(doc) +} + +/// The BIFF stream: `Workbook` (BIFF8) or `Book` (older BIFF), matched +/// case-insensitively like detection because producers vary. +fn workbook_stream( + ole: &mut cfb::CompoundFile, +) -> Result, ConvertError> { + let name = ole + .read_root_storage() + .find(|e| { + e.is_stream() + && (e.name().eq_ignore_ascii_case("Workbook") + || e.name().eq_ignore_ascii_case("Book")) + }) + .map(|e| e.name().to_string()) + .ok_or(ConvertError::MissingPart { part: "Workbook".to_string() })?; + read_ole_stream(ole, &name) +} + +/// The record at `pos`: type, payload, and the position after it. `None` on +/// a truncated header or payload. +fn record_at(data: &[u8], pos: usize) -> Option<(u16, &[u8], usize)> { + let rec_type = get_u16(data, pos)?; + let len = get_u16(data, pos.checked_add(2)?)? as usize; + let body_at = pos.checked_add(4)?; + let body = data.get(body_at..body_at.checked_add(len)?)?; + Some((rec_type, body, body_at + len)) +} + +/// A record: type, payload, position after it. +type Record<'a> = (u16, &'a [u8], usize); + +/// `record_at` charging the stream-wide record budget. +fn next_record<'a>( + data: &'a [u8], + pos: usize, + records: &mut u64, +) -> Result>, ConvertError> { + let Some(rec) = record_at(data, pos) else { + // Trailing bytes too short to form a record: the stream was cut, so + // whatever was read is partial rather than complete. + if pos < data.len() { + log::warn!("workbook stream ends mid-record at byte {pos}; output may be partial"); + } + return Ok(None); + }; + *records += 1; + if *records > limits::MAX_RECORDS { + return Err(ConvertError::ResourceLimit { + limit: "max_records", + detail: format!("workbook stream exceeds {} records", limits::MAX_RECORDS), + }); + } + Ok(Some(rec)) +} + +/// The record body at `pos - body.len() - 4` plus the bodies of the +/// CONTINUE records that immediately follow it, and the position after them. +fn continued<'a>( + data: &'a [u8], + body: &'a [u8], + mut pos: usize, + records: &mut u64, +) -> Result<(Vec<&'a [u8]>, usize), ConvertError> { + let mut segs = vec![body]; + while let Some((rec_type, cont, next)) = next_record(data, pos, records)? { + if rec_type != CONTINUE { + *records -= 1; + break; + } + segs.push(cont); + pos = next; + } + Ok((segs, pos)) +} + +/// Reader over a record's segments (base body plus CONTINUE bodies). Fixed +/// fields never straddle a segment boundary, but may start exactly on one; +/// only string character data crosses boundaries, and the string readers +/// handle the repeated option-flags byte themselves. +struct SegReader<'a> { + segs: Vec<&'a [u8]>, + seg: usize, + off: usize, +} + +impl<'a> SegReader<'a> { + fn new(segs: Vec<&'a [u8]>) -> SegReader<'a> { + SegReader { segs, seg: 0, off: 0 } + } + + /// Bytes left in the current segment. + fn in_seg(&self) -> usize { + self.segs.get(self.seg).map_or(0, |s| s.len() - self.off) + } + + /// Hop over exhausted segments so a field can start at a boundary. + fn normalize(&mut self) { + while self.seg < self.segs.len() && self.in_seg() == 0 { + self.seg += 1; + self.off = 0; + } + } + + /// Move to the start of the next segment; `None` when there is none. + fn next_seg(&mut self) -> Option<()> { + if self.seg + 1 < self.segs.len() { + self.seg += 1; + self.off = 0; + Some(()) + } else { + None + } + } + + /// `n` bytes from within one segment. + fn bytes(&mut self, n: usize) -> Option<&'a [u8]> { + self.normalize(); + let seg = self.segs.get(self.seg)?; + let out = seg.get(self.off..self.off.checked_add(n)?)?; + self.off += n; + Some(out) + } + + fn u8(&mut self) -> Option { + self.bytes(1).map(|b| b[0]) + } + + fn u16(&mut self) -> Option { + self.bytes(2).map(|b| u16::from_le_bytes([b[0], b[1]])) + } + + fn u32(&mut self) -> Option { + self.bytes(4).and_then(|b| Some(u32::from_le_bytes(b.try_into().ok()?))) + } + + /// Skip `n` bytes across segment boundaries (non-character data carries + /// no repeated flags byte). + fn skip(&mut self, mut n: usize) -> Option<()> { + while n > 0 { + self.normalize(); + let step = self.in_seg().min(n); + if step == 0 { + return None; + } + self.off += step; + n -= step; + } + Some(()) + } +} + +/// A BIFF8 Unicode string: XLUnicodeString, ShortXLUnicodeString (`short`), +/// or XLUnicodeRichExtendedString (`rich`). Character data may continue +/// into following segments; at every such boundary the option-flags byte is +/// repeated and the encoding can switch between 8-bit compressed and 16-bit +/// UTF-16, so it is re-read rather than carried over. Rich runs and +/// phonetic data are skipped, matching the xlsx reader's handling of `rPh`. +fn read_biff8_string(r: &mut SegReader, short: bool, rich: bool) -> Option { + let cch = if short { r.u8()? as usize } else { r.u16()? as usize }; + let flags = r.u8()?; + let mut wide = flags & 0x01 != 0; + let runs = if rich && flags & 0x08 != 0 { r.u16()? as usize } else { 0 }; + let ext = if rich && flags & 0x04 != 0 { r.u32()? as usize } else { 0 }; + let mut units: Vec = Vec::new(); + let mut remaining = cch; + while remaining > 0 { + if r.in_seg() == 0 { + r.next_seg()?; + wide = r.u8()? & 0x01 != 0; + } + let unit = if wide { 2 } else { 1 }; + let take = (r.in_seg() / unit).min(remaining); + if take == 0 { + // A dangling half character: outside the format. + return None; + } + let bytes = r.bytes(take * unit)?; + if wide { + units.extend(bytes.chunks_exact(2).map(|c| u16::from_le_bytes([c[0], c[1]]))); + } else { + units.extend(bytes.iter().map(|&b| u16::from(b))); + } + remaining -= take; + } + // A truncated trailer loses only the strings after this one; the reader + // then runs dry and the caller stops. + let _ = r.skip(runs * 4).and_then(|()| r.skip(ext)); + Some(String::from_utf16_lossy(&units)) +} + +/// A BIFF5/BIFF7 byte string (no flags byte), decoded per the workbook's +/// CODEPAGE record. +fn read_byte_string( + r: &mut SegReader, + short: bool, + encoding: &'static encoding_rs::Encoding, +) -> Option { + let cch = if short { r.u8()? as usize } else { r.u16()? as usize }; + let mut bytes = Vec::with_capacity(cch); + let mut remaining = cch; + while remaining > 0 { + r.normalize(); + let take = r.in_seg().min(remaining); + if take == 0 { + return None; + } + bytes.extend_from_slice(r.bytes(take)?); + remaining -= take; + } + let (text, _) = encoding.decode_without_bom_handling(&bytes); + Some(text.into_owned()) +} + +/// ANSI code page from the CODEPAGE record, for BIFF5 byte strings (BIFF8 +/// strings carry their own encoding flag). +fn codepage_encoding(cp: u16) -> &'static encoding_rs::Encoding { + use encoding_rs::*; + match cp { + 874 => WINDOWS_874, + 932 => SHIFT_JIS, + 936 => GBK, + 949 => EUC_KR, + 950 => BIG5, + 1250 => WINDOWS_1250, + 1251 => WINDOWS_1251, + 1253 => WINDOWS_1253, + 1254 => WINDOWS_1254, + 1255 => WINDOWS_1255, + 1256 => WINDOWS_1256, + 1257 => WINDOWS_1257, + 1258 => WINDOWS_1258, + _ => WINDOWS_1252, + } +} + +struct BoundSheet { + name: String, + /// Absolute stream offset of the sheet substream's BOF record. + offset: usize, + visible: bool, +} + +struct Globals { + biff8: bool, + date1904: bool, + encoding: &'static encoding_rs::Encoding, + sst: Vec, + /// The XF table in record order; a cell's ixfe indexes it directly. + xfs: Vec, + sheets: Vec, +} + +impl Globals { + fn format(&self, ixfe: u16) -> &CellFormat { + self.xfs.get(usize::from(ixfe)).unwrap_or(&CellFormat::General) + } + + fn read_string(&self, r: &mut SegReader, short: bool) -> Option { + if self.biff8 { + read_biff8_string(r, short, false) + } else { + read_byte_string(r, short, self.encoding) + } + } +} + +/// The workbook globals substream: SST, FORMAT/XF tables, sheet directory, +/// date system, encryption marker. +fn read_globals(data: &[u8], records: &mut u64) -> Result { + let Some((rec_type, body, mut pos)) = record_at(data, 0) else { + return Err(ConvertError::malformed("empty workbook stream")); + }; + if rec_type != BOF { + return Err(ConvertError::malformed("workbook stream does not start with a BOF record")); + } + let mut globals = Globals { + biff8: get_u16(body, 0) == Some(0x0600), + date1904: false, + encoding: encoding_rs::WINDOWS_1252, + sst: Vec::new(), + xfs: Vec::new(), + sheets: Vec::new(), + }; + let mut formats: HashMap = HashMap::new(); + let mut xf_ifmts: Vec = Vec::new(); + let mut depth = 1usize; + while let Some((rec_type, body, next)) = next_record(data, pos, records)? { + pos = next; + match rec_type { + BOF => depth += 1, + EOF_REC => { + if depth == 1 { + break; + } + depth -= 1; + } + _ if depth > 1 => {} + FILEPASS => return Err(ConvertError::Encrypted), + CODEPAGE => { + if let Some(cp) = get_u16(body, 0) { + globals.encoding = codepage_encoding(cp); + } + } + DATEMODE => globals.date1904 = get_u16(body, 0) == Some(1), + BOUNDSHEET => { + if let Some(sheet) = read_boundsheet(body, &globals) { + globals.sheets.push(sheet); + } + } + FORMAT => { + let Some(ifmt) = get_u16(body, 0) else { + continue; + }; + let mut r = SegReader::new(vec![&body[2..]]); + // BIFF5 FORMAT carries a one-byte length, BIFF8 a two-byte. + if let Some(code) = globals.read_string(&mut r, !globals.biff8) { + formats.insert(u32::from(ifmt), code); + } + } + XF => { + // ixfe is 16-bit, so the table never usefully exceeds it. + if xf_ifmts.len() <= usize::from(u16::MAX) + && let Some(ifmt) = get_u16(body, 2) + { + xf_ifmts.push(ifmt); + } + } + SST if globals.biff8 => { + let (segs, after) = continued(data, body, pos, records)?; + globals.sst = read_sst(&segs); + pos = after; + } + _ => {} + } + } + // Resolved after the pass: FORMAT records are not ordered relative to + // the XFs that reference them. + let custom: HashMap = + formats.iter().map(|(&id, code)| (id, code.as_str())).collect(); + let mut cache: HashMap = HashMap::new(); + globals.xfs = xf_ifmts + .iter() + .map(|&ifmt| { + cache.entry(ifmt).or_insert_with(|| resolve_format(u32::from(ifmt), &custom)).clone() + }) + .collect(); + Ok(globals) +} + +/// BOUNDSHEET: substream offset, hidden state, sheet type, name. Chart and +/// macro sheets stay listed (their substreams fail the worksheet check and +/// count as unreadable, matching the xlsx reader's treatment of their +/// parts); VBA modules are not sheets in any container and are dropped. +fn read_boundsheet(body: &[u8], globals: &Globals) -> Option { + let offset = get_u32(body, 0)? as usize; + let state = body.get(4)? & 0x03; + if *body.get(5)? == 0x06 { + return None; + } + let mut r = SegReader::new(vec![body.get(6..)?]); + let name = globals.read_string(&mut r, true)?; + Some(BoundSheet { name: clean_text(&name), offset, visible: state == 0 }) +} + +/// The shared string table (trap 1: strings routinely split across +/// CONTINUE records, mid-character-data). A malformed tail keeps the +/// entries read so far; LABELSST lookups past them log and stay empty. +fn read_sst(segs: &[&[u8]]) -> Vec { + let mut r = SegReader::new(segs.to_vec()); + let unique = match (r.u32(), r.u32()) { + (Some(_total), Some(unique)) => unique as usize, + _ => return Vec::new(), + }; + let mut out = Vec::new(); + while out.len() < unique { + let Some(text) = read_biff8_string(&mut r, false, true) else { + if out.len() < unique { + log::debug!("shared string table truncated at entry {}", out.len()); + } + break; + }; + out.push(clean_text(&text)); + } + out +} + +/// A cell record's leading (row, column, ixfe); `None` when the column is +/// outside the grid the format defines. +fn cell_ref(body: &[u8]) -> Option<(u32, u32, u16)> { + let row = u32::from(get_u16(body, 0)?); + let col = u32::from(get_u16(body, 2)?); + let ixfe = get_u16(body, 4)?; + (col < MAX_COLS).then_some((row, col, ixfe)) +} + +fn get_f64(body: &[u8], off: usize) -> Option { + Some(f64::from_le_bytes(body.get(off..off.checked_add(8)?)?.try_into().ok()?)) +} + +/// One worksheet substream into the shared `SheetContent` shape. `Ok(None)` +/// means the substream is missing or not a worksheet (chart or macro +/// sheets), which the caller counts as unreadable. +fn read_sheet( + data: &[u8], + globals: &Globals, + offset: usize, + records: &mut u64, +) -> Result, ConvertError> { + let Some((rec_type, body, mut pos)) = record_at(data, offset) else { + return Ok(None); + }; + if rec_type != BOF || get_u16(body, 2) != Some(WORKSHEET_SUBSTREAM) { + return Ok(None); + } + let mut out = SheetContent::default(); + let mut depth = 1usize; + // A FORMULA whose cached value is a string: (row, col, ixfe) waiting + // for the STRING record that carries the text. + let mut pending: Option<(u32, u32, u16)> = None; + while let Some((rec_type, body, next)) = next_record(data, pos, records)? { + pos = next; + match rec_type { + BOF => depth += 1, + EOF_REC => { + if depth == 1 { + break; + } + depth -= 1; + } + _ if depth > 1 => {} + ROW => { + if body.get(12).is_some_and(|flags| flags & 0x20 != 0) + && let Some(row) = get_u16(body, 0) + { + out.hidden_rows.insert(u32::from(row)); + } + } + COLINFO => { + if let (Some(first), Some(last), Some(flags)) = + (get_u16(body, 0), get_u16(body, 2), get_u16(body, 8)) + && flags & 0x01 != 0 + && first <= last + { + out.hidden_cols.push((u32::from(first), u32::from(last).min(MAX_COLS - 1))); + } + } + MERGEDCELLS => { + let count = usize::from(get_u16(body, 0).unwrap_or(0)) + .min(body.len().saturating_sub(2) / 8); + for i in 0..count { + let at = 2 + i * 8; + let (Some(r1), Some(r2), Some(c1), Some(c2)) = ( + get_u16(body, at), + get_u16(body, at + 2), + get_u16(body, at + 4), + get_u16(body, at + 6), + ) else { + break; + }; + let (r1, r2) = (u32::from(r1.min(r2)), u32::from(r1.max(r2))); + let (c1, c2) = (u32::from(c1.min(c2)), u32::from(c1.max(c2))); + if c1 >= MAX_COLS || (r1 == r2 && c1 == c2) { + continue; + } + out.merges.push((r1, c1, r2, c2.min(MAX_COLS - 1))); + } + } + LABELSST => { + if let Some((row, col, ixfe)) = cell_ref(body) + && let Some(isst) = get_u32(body, 6) + { + match globals.sst.get(isst as usize) { + Some(text) => { + put(&mut out, row, col, format_as_text(globals.format(ixfe), text)); + } + None => log::debug!("shared string index {isst} out of range"), + } + } + } + LABEL | RSTRING => { + let (segs, after) = continued(data, body, pos, records)?; + pos = after; + if let Some((row, col, ixfe)) = cell_ref(body) + && let Some(mut r) = string_reader(&segs, 6) + && let Some(text) = globals.read_string(&mut r, false) + { + // RSTRING's trailing rich runs are formatting, not text. + put( + &mut out, + row, + col, + format_as_text(globals.format(ixfe), &clean_text(&text)), + ); + } + } + NUMBER => { + if let Some((row, col, ixfe)) = cell_ref(body) + && let Some(n) = get_f64(body, 6) + { + put( + &mut out, + row, + col, + render_number(globals.format(ixfe), n, globals.date1904), + ); + } + } + RK => { + if let Some((row, col, ixfe)) = cell_ref(body) + && let Some(rk) = get_u32(body, 6) + { + let n = rk_number(rk); + put( + &mut out, + row, + col, + render_number(globals.format(ixfe), n, globals.date1904), + ); + } + } + MULRK => { + // rw, colFirst, then 6-byte (ixfe, RK) pairs; colLast is + // redundant with the record length. + if let (Some(row), Some(col_first)) = (get_u16(body, 0), get_u16(body, 2)) { + for i in 0..body.len().saturating_sub(6) / 6 { + let (Some(ixfe), Some(rk)) = + (get_u16(body, 4 + i * 6), get_u32(body, 6 + i * 6)) + else { + break; + }; + let col = u32::from(col_first) + i as u32; + if col >= MAX_COLS { + break; + } + let n = rk_number(rk); + let text = render_number(globals.format(ixfe), n, globals.date1904); + put(&mut out, u32::from(row), col, text); + } + } + } + BOOLERR => { + if let Some((row, col, _ixfe)) = cell_ref(body) + && let (Some(&value), Some(&is_err)) = (body.get(6), body.get(7)) + { + let text = match (is_err, value) { + (0, 0) => Some("FALSE"), + (0, _) => Some("TRUE"), + (1, code) => error_literal(code), + _ => None, + }; + if let Some(text) = text { + put(&mut out, row, col, text.to_string()); + } + } + } + FORMULA => { + if let Some((row, col, ixfe)) = cell_ref(body) + && let Some(value) = body.get(6..14) + { + if value[6] == 0xFF && value[7] == 0xFF { + match value[0] { + 0x00 => pending = Some((row, col, ixfe)), + 0x01 => { + let text = if value[2] == 0 { "FALSE" } else { "TRUE" }; + put(&mut out, row, col, text.to_string()); + } + 0x02 => { + if let Some(text) = error_literal(value[2]) { + put(&mut out, row, col, text.to_string()); + } + } + // 0x03 is a blank string result. + _ => {} + } + } else if let Some(n) = get_f64(body, 6) { + let text = render_number(globals.format(ixfe), n, globals.date1904); + put(&mut out, row, col, text); + } + } + } + STRING => { + let (segs, after) = continued(data, body, pos, records)?; + pos = after; + if let Some((row, col, ixfe)) = pending.take() + && let Some(mut r) = string_reader(&segs, 0) + && let Some(text) = globals.read_string(&mut r, false) + { + put( + &mut out, + row, + col, + format_as_text(globals.format(ixfe), &clean_text(&text)), + ); + } + } + _ => {} + } + } + Ok(Some(out)) +} + +/// Record only non-empty cell text, like the xlsx reader. +fn put(out: &mut SheetContent, row: u32, col: u32, text: String) { + if !text.is_empty() { + out.cells.insert((row, col), text); + } +} + +/// A reader over `segs` with the first segment's leading `skip` bytes (the +/// cell header before the string) removed. +fn string_reader<'a>(segs: &'a [&'a [u8]], skip: usize) -> Option> { + let mut parts: Vec<&[u8]> = Vec::with_capacity(segs.len()); + parts.push(segs.first()?.get(skip..)?); + parts.extend(&segs[1..]); + Some(SegReader::new(parts)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{CellSlot, Table, inlines_to_plain_text}; + use std::io::Write; + + fn rec(rec_type: u16, body: &[u8]) -> Vec { + let mut out = rec_type.to_le_bytes().to_vec(); + out.extend((body.len() as u16).to_le_bytes()); + out.extend_from_slice(body); + out + } + + fn bof(dt: u16, vers: u16) -> Vec { + let mut body = vec![0u8; 16]; + body[..2].copy_from_slice(&vers.to_le_bytes()); + body[2..4].copy_from_slice(&dt.to_le_bytes()); + rec(BOF, &body) + } + + /// XLUnicodeString, compressed ASCII. + fn ustr(s: &str) -> Vec { + let mut out = (s.len() as u16).to_le_bytes().to_vec(); + out.push(0); + out.extend_from_slice(s.as_bytes()); + out + } + + /// ShortXLUnicodeString, compressed ASCII. + fn short_ustr(s: &str) -> Vec { + let mut out = vec![s.len() as u8, 0]; + out.extend_from_slice(s.as_bytes()); + out + } + + fn cell6(row: u16, col: u16, ixfe: u16) -> Vec { + [row, col, ixfe].iter().flat_map(|v| v.to_le_bytes()).collect() + } + + fn labelsst(row: u16, col: u16, ixfe: u16, isst: u32) -> Vec { + let mut body = cell6(row, col, ixfe); + body.extend(isst.to_le_bytes()); + rec(LABELSST, &body) + } + + fn label(row: u16, col: u16, ixfe: u16, s: &str) -> Vec { + let mut body = cell6(row, col, ixfe); + body.extend(ustr(s)); + rec(LABEL, &body) + } + + fn number(row: u16, col: u16, ixfe: u16, v: f64) -> Vec { + let mut body = cell6(row, col, ixfe); + body.extend(v.to_le_bytes()); + rec(NUMBER, &body) + } + + fn rk_cell(row: u16, col: u16, ixfe: u16, rk: u32) -> Vec { + let mut body = cell6(row, col, ixfe); + body.extend(rk.to_le_bytes()); + rec(RK, &body) + } + + fn formula(row: u16, col: u16, ixfe: u16, value: [u8; 8]) -> Vec { + let mut body = cell6(row, col, ixfe); + body.extend(value); + body.extend([0u8; 6]); + rec(FORMULA, &body) + } + + fn rk_from_f64(v: f64) -> u32 { + ((v.to_bits() >> 32) as u32) & !3 + } + + fn rk_from_int(v: i32) -> u32 { + ((v as u32) << 2) | 2 + } + + fn ole_with(name: &str, data: &[u8]) -> Vec { + let mut ole = cfb::CompoundFile::create(Cursor::new(Vec::new())).unwrap(); + ole.create_stream(name).unwrap().write_all(data).unwrap(); + ole.into_inner().into_inner() + } + + /// Assemble a BIFF8 workbook: globals plus one substream per sheet, + /// with each BOUNDSHEET's lbPlyPos patched to the real offset. + #[derive(Default)] + struct Wb { + date1904: bool, + filepass: bool, + /// FORMAT records: (ifmt, code). + formats: Vec<(u16, &'static str)>, + /// XF records in table order, each carrying its ifmt. + xfs: Vec, + /// Raw SST records (the SST plus any CONTINUE records). + sst: Vec>, + /// (name, hsState, substream records without BOF/EOF). + sheets: Vec<(&'static str, u8, Vec)>, + } + + impl Wb { + fn build(&self) -> Vec { + let mut stream = bof(0x0005, 0x0600); + if self.filepass { + stream.extend(rec(FILEPASS, &[0u8; 6])); + } + if self.date1904 { + stream.extend(rec(DATEMODE, &1u16.to_le_bytes())); + } + for (ifmt, code) in &self.formats { + let mut body = ifmt.to_le_bytes().to_vec(); + body.extend(ustr(code)); + stream.extend(rec(FORMAT, &body)); + } + for ifmt in &self.xfs { + let mut body = vec![0u8; 20]; + body[2..4].copy_from_slice(&ifmt.to_le_bytes()); + stream.extend(rec(XF, &body)); + } + for raw in &self.sst { + stream.extend(raw); + } + let mut patch_at = Vec::new(); + for (name, state, _) in &self.sheets { + let mut body = vec![0u8; 4]; + body.push(*state); + body.push(0); + body.extend(short_ustr(name)); + patch_at.push(stream.len() + 4); + stream.extend(rec(BOUNDSHEET, &body)); + } + stream.extend(rec(EOF_REC, &[])); + for (i, (_, _, records)) in self.sheets.iter().enumerate() { + let offset = (stream.len() as u32).to_le_bytes(); + stream[patch_at[i]..patch_at[i] + 4].copy_from_slice(&offset); + stream.extend(bof(WORKSHEET_SUBSTREAM, 0x0600)); + stream.extend_from_slice(records); + stream.extend(rec(EOF_REC, &[])); + } + ole_with("Workbook", &stream) + } + } + + fn one_sheet(records: Vec) -> Wb { + Wb { xfs: vec![0], sheets: vec![("S", 0, records)], ..Wb::default() } + } + + fn first_table(doc: &Document) -> &Table { + match doc.blocks.iter().find_map(|b| match b { + Block::Table(t) => Some(t), + _ => None, + }) { + Some(t) => t, + None => panic!("expected a table, got {:?}", doc.blocks), + } + } + + fn texts(table: &Table) -> Vec> { + table + .grid + .iter() + .map(|row| { + row.iter() + .map(|slot| match slot { + CellSlot::Origin(cell) => cell + .blocks + .iter() + .filter_map(|b| match b { + Block::Paragraph(i) => Some(inlines_to_plain_text(i)), + _ => None, + }) + .collect(), + CellSlot::Covered { .. } => "".to_string(), + }) + .collect() + }) + .collect() + } + + #[test] + fn sst_string_split_mid_word_re_reads_the_flags_byte() { + // Trap 1: the string continues into a CONTINUE record and switches + // from 8-bit compressed to 16-bit encoding at the boundary, marked + // by the repeated option-flags byte. + let mut base = 1u32.to_le_bytes().to_vec(); + base.extend(1u32.to_le_bytes()); + base.extend(11u16.to_le_bytes()); + base.push(0x00); + base.extend(b"HELLO"); + let mut cont = vec![0x01]; + for unit in " WORLD".encode_utf16() { + cont.extend(unit.to_le_bytes()); + } + let wb = Wb { + sst: vec![rec(SST, &base), rec(CONTINUE, &cont)], + ..one_sheet(labelsst(0, 0, 0, 0)) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["HELLO WORLD"]]); + } + + #[test] + fn rich_and_phonetic_headers_do_not_misalign_the_table() { + // Trap 2: the optional rich-run count and phonetic size headers and + // their trailing data must be consumed, or the next string reads + // from the middle of them. + let mut body = 2u32.to_le_bytes().to_vec(); + body.extend(2u32.to_le_bytes()); + body.extend(2u16.to_le_bytes()); + body.push(0x0C); // fRichSt | fExtSt + body.extend(1u16.to_le_bytes()); // cRun + body.extend(4u32.to_le_bytes()); // cbExtRst + body.extend(b"ab"); + body.extend([0xAA; 4]); // the rich run + body.extend([0xBB; 4]); // the phonetic block + body.extend(ustr("ok")); + let mut records = labelsst(0, 0, 0, 0); + records.extend(labelsst(0, 1, 0, 1)); + let wb = Wb { sst: vec![rec(SST, &body)], ..one_sheet(records) }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["ab", "ok"]]); + } + + #[test] + fn rk_and_mulrk_encodings_decode() { + // Trap 3: RK packs an integer or a truncated double, with a + // divide-by-100 bit; MULRK packs a run of them, each with its own + // format index. + let mut records = rk_cell(0, 0, 0, rk_from_f64(1234.5)); + records.extend(rk_cell(0, 1, 0, rk_from_int(15550) | 1)); // 155.5 + let mut mulrk = cell6(0, 2, 0)[..4].to_vec(); // rw, colFirst + mulrk.extend(0u16.to_le_bytes()); // ixfe General + mulrk.extend(rk_from_int(7).to_le_bytes()); + mulrk.extend(1u16.to_le_bytes()); // ixfe of the percent XF + mulrk.extend(rk_from_f64(0.5).to_le_bytes()); + mulrk.extend(3u16.to_le_bytes()); // colLast + records.extend(rec(MULRK, &mulrk)); + let wb = Wb { + formats: vec![(164, "0%")], + xfs: vec![0, 164], + sheets: vec![("S", 0, records)], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["1234.5", "155.5", "7", "50%"]]); + } + + #[test] + fn number_formats_apply_to_numeric_cells() { + // Trap 4: the cell's ixfe indexes the XF table, whose ifmt resolves + // against FORMAT records first and the built-in table second (ifmt + // 3 has no FORMAT record here). + let mut records = number(0, 0, 1, 0.155); + records.extend(number(0, 1, 2, 1234.5)); + records.extend(number(0, 2, 3, 9876543.0)); + let wb = Wb { + formats: vec![(164, "0.0%"), (165, "\"$\"#,##0.00")], + xfs: vec![0, 164, 165, 3], + sheets: vec![("S", 0, records)], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["15.5%", "$1,234.50", "9,876,543"]]); + } + + #[test] + fn date_serials_render_iso_in_both_date_systems() { + let dated = |date1904| Wb { + date1904, + formats: vec![(164, "yyyy-mm-dd")], + xfs: vec![164], + sheets: vec![("S", 0, number(0, 0, 0, if date1904 { 100.0 } else { 46096.0 }))], + ..Wb::default() + }; + let doc = parse(&dated(false).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["2026-03-15"]]); + let doc = parse(&dated(true).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["1904-04-10"]]); + } + + #[test] + fn hidden_rows_columns_and_sheets_are_omitted() { + let mut records = label(0, 0, 0, "a"); + records.extend(label(0, 1, 0, "hidden col")); + records.extend(label(0, 2, 0, "c")); + records.extend(label(1, 0, 0, "hidden row")); + records.extend(label(2, 0, 0, "d")); + let mut row = vec![0u8; 16]; + row[..2].copy_from_slice(&1u16.to_le_bytes()); + row[12] = 0x20; // fDyZero + records.extend(rec(ROW, &row)); + let mut colinfo = vec![0u8; 12]; + colinfo[..2].copy_from_slice(&1u16.to_le_bytes()); + colinfo[2..4].copy_from_slice(&1u16.to_le_bytes()); + colinfo[8] = 0x01; // fHidden + records.extend(rec(COLINFO, &colinfo)); + let wb = Wb { + xfs: vec![0], + sheets: vec![("Shown", 0, records), ("Secret", 1, label(0, 0, 0, "secret"))], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(doc.blocks.len(), 1, "hidden sheet must add no heading and no table"); + assert_eq!(texts(first_table(&doc)), vec![vec!["a", "c"], vec!["d", ""]]); + } + + #[test] + fn merge_extends_past_the_populated_range() { + // Issue #8: the only populated cell anchors F1:O3, so the grid must + // widen to the merge's full 3x10 extent. + let mut records = label(0, 5, 0, "wide"); + let mut merged = 1u16.to_le_bytes().to_vec(); + for v in [0u16, 2, 5, 14] { + merged.extend(v.to_le_bytes()); + } + records.extend(rec(MERGEDCELLS, &merged)); + let doc = parse(&one_sheet(records).build()).unwrap(); + let table = first_table(&doc); + assert_eq!(table.grid.len(), 3); + assert_eq!(table.grid[1].len(), 10); + let CellSlot::Origin(cell) = &table.grid[0][0] else { + panic!("expected the merge origin at (0,0)"); + }; + assert_eq!((cell.col_span, cell.row_span), (10, 3)); + } + + #[test] + fn formula_cached_values_render() { + let mut records = formula(0, 0, 0, 2.5f64.to_le_bytes()); + records.extend(formula(0, 1, 0, [0x00, 0, 0, 0, 0, 0, 0xFF, 0xFF])); + records.extend(rec(STRING, &ustr("calc"))); + records.extend(formula(0, 2, 0, [0x01, 0, 1, 0, 0, 0, 0xFF, 0xFF])); + records.extend(formula(0, 3, 0, [0x02, 0, 0x17, 0, 0, 0, 0xFF, 0xFF])); + let doc = parse(&one_sheet(records).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["2.5", "calc", "TRUE", "#REF!"]]); + } + + #[test] + fn boolerr_renders_bool_and_error_literals() { + let cell = |col: u16, value: u8, is_err: u8| { + let mut body = cell6(0, col, 0); + body.extend([value, is_err]); + rec(BOOLERR, &body) + }; + let mut records = cell(0, 1, 0); + records.extend(cell(1, 0, 0)); + records.extend(cell(2, 0x07, 1)); + let doc = parse(&one_sheet(records).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["TRUE", "FALSE", "#DIV/0!"]]); + } + + #[test] + fn filepass_means_encrypted() { + let wb = Wb { filepass: true, ..one_sheet(label(0, 0, 0, "x")) }; + assert!(matches!(parse(&wb.build()), Err(ConvertError::Encrypted))); + } + + #[test] + fn biff5_byte_strings_degrade_gracefully() { + // A BIFF5 stream in a `Book` container: one-byte-length boundsheet + // names and codepage-encoded LABEL text, no SST. + let mut stream = bof(0x0005, 0x0500); + stream.extend(rec(CODEPAGE, &1252u16.to_le_bytes())); + let mut xf = vec![0u8; 16]; + xf[2..4].copy_from_slice(&0u16.to_le_bytes()); + stream.extend(rec(XF, &xf)); + let mut boundsheet = vec![0u8; 6]; + boundsheet.extend([1, b'S']); + let patch_at = stream.len() + 4; + stream.extend(rec(BOUNDSHEET, &boundsheet)); + stream.extend(rec(EOF_REC, &[])); + let offset = (stream.len() as u32).to_le_bytes(); + stream[patch_at..patch_at + 4].copy_from_slice(&offset); + stream.extend(bof(WORKSHEET_SUBSTREAM, 0x0500)); + let mut body = cell6(0, 0, 0); + body.extend(6u16.to_le_bytes()); + body.extend(b"l\xE9gacy"); // cp1252 e-acute + stream.extend(rec(LABEL, &body)); + stream.extend(rec(EOF_REC, &[])); + let doc = parse(&ole_with("Book", &stream)).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["l\u{e9}gacy"]]); + } +} diff --git a/src/formats/sheet/xlsb.rs b/src/formats/sheet/xlsb.rs new file mode 100644 index 00000000..5f3beedf --- /dev/null +++ b/src/formats/sheet/xlsb.rs @@ -0,0 +1,863 @@ +//! In-house binary SpreadsheetML reader (.xlsb): the same OPC package as +//! xlsx with binary record streams (MS-XLSB) in place of XML parts. Cell +//! values, number formats, visibility and merges are decoded here; grid +//! materialization, format resolution and date rendering are shared with +//! the xlsx reader so both containers convert identically. + +use super::xlsx::{ + CellFormat, MAX_COLS, MAX_ROWS, SHARED_STRINGS_REL, SheetContent, build_table, format_as_text, + render_number, resolve_format, sibling_part_name, +}; +use super::{error_literal, rk_number}; +use crate::error::ConvertError; +use crate::model::{Block, Document, Inline}; +use crate::package::limits; +use crate::package::relationships::{read_rels, rel_type, rels_part_for}; +use crate::package::{Package, path}; +use crate::shared::text::clean_text; +use std::collections::HashMap; + +// MS-XLSB record type values (section 2.3, "By Number"). +const BRT_ROW_HDR: u16 = 0; +const BRT_CELL_RK: u16 = 2; +const BRT_CELL_ERROR: u16 = 3; +const BRT_CELL_BOOL: u16 = 4; +const BRT_CELL_REAL: u16 = 5; +const BRT_CELL_ST: u16 = 6; +const BRT_CELL_ISST: u16 = 7; +const BRT_FMLA_STRING: u16 = 8; +const BRT_FMLA_NUM: u16 = 9; +const BRT_FMLA_BOOL: u16 = 10; +const BRT_FMLA_ERROR: u16 = 11; +const BRT_SST_ITEM: u16 = 19; +const BRT_FMT: u16 = 44; +const BRT_XF: u16 = 47; +const BRT_COL_INFO: u16 = 60; +const BRT_CELL_RSTRING: u16 = 62; +const BRT_WB_PROP: u16 = 153; +const BRT_BUNDLE_SH: u16 = 156; +const BRT_MERGE_CELL: u16 = 176; +const BRT_BEGIN_CELL_XFS: u16 = 617; +const BRT_END_CELL_XFS: u16 = 618; + +pub(super) fn parse(pkg: &mut Package, wb_part: &str) -> Result { + let workbook = pkg.required_part(wb_part)?; + let (date1904, bundles) = read_workbook(&workbook)?; + let wb_rels = read_rels(pkg, &rels_part_for(wb_part))?; + + let shared = read_optional( + pkg, + &sibling_part_name(&wb_rels, wb_part, SHARED_STRINGS_REL, "sharedStrings.bin"), + read_shared_strings, + )?; + let xfs = read_optional( + pkg, + &sibling_part_name(&wb_rels, wb_part, rel_type::STYLES, "styles.bin"), + read_styles, + )?; + + // Visible sheets in workbook order, resolved to their parts exactly as + // in xlsx: by relationship id, never by conventional part name. + let mut sheets: Vec<(String, String)> = Vec::new(); + for (name, rid) in bundles { + let Some(target) = wb_rels.internal_target(&rid) else { + log::warn!("skipping sheet {name:?} with no worksheet relationship"); + continue; + }; + match path::resolve(wb_part, target) { + Ok(t) => sheets.push((name, t.path)), + Err(e) => log::warn!("skipping sheet {name:?} with unresolvable target: {e}"), + } + } + + let multi_sheet = sheets.len() > 1; + let mut doc = Document::default(); + let mut failed = 0usize; + // One budget for the workbook, so sheets cannot multiply the cap. + let mut slots = 0u64; + for (name, part) in &sheets { + let content = + pkg.optional_part(part)?.map(|bytes| read_sheet(&bytes, &shared, &xfs, date1904)); + let content = match content { + Some(Ok(c)) => c, + Some(Err(e)) if e.is_fatal() => return Err(e), + Some(Err(e)) => { + log::warn!("skipping unreadable sheet {name:?}: {e}"); + failed += 1; + continue; + } + None => { + log::warn!("skipping unreadable sheet {name:?}"); + failed += 1; + continue; + } + }; + let Some(table) = build_table(content, &mut slots)? else { + continue; + }; + if multi_sheet { + doc.blocks.push(Block::heading(2, vec![Inline::plain(name.clone())])); + } + doc.blocks.push(Block::Table(table)); + } + if !sheets.is_empty() && failed == sheets.len() { + return Err(ConvertError::malformed("no sheet in the workbook could be read")); + } + Ok(doc) +} + +/// Read an optional binary part under the unified recovery policy: absent or +/// corrupt yields the default with a log; fatal resource-limit errors always +/// propagate. +fn read_optional( + pkg: &mut Package, + part: &str, + read: fn(&[u8]) -> Result, +) -> Result { + let Some(bytes) = pkg.optional_part(part)? else { + return Ok(T::default()); + }; + match read(&bytes) { + Ok(v) => Ok(v), + Err(e) if e.is_fatal() => Err(e), + Err(e) => { + log::warn!("skipping corrupt part {part}: {e}"); + Ok(T::default()) + } + } +} + +/// `xl/workbook.bin`: the 1904 date flag from BrtWbProp, and each visible +/// sheet's name and relationship id from its BrtBundleSh. +fn read_workbook(data: &[u8]) -> Result<(bool, Vec<(String, String)>), ConvertError> { + let mut date1904 = false; + let mut sheets = Vec::new(); + let mut records = Records::new(data); + while let Some((id, payload)) = records.next()? { + match id { + BRT_WB_PROP => date1904 = Fields::new(payload).u32()? & 1 != 0, + BRT_BUNDLE_SH => { + let mut f = Fields::new(payload); + let state = f.u32()?; + f.u32()?; // iTabID + let rid = f.nullable_wide_string()?; + let name = clean_text(&f.wide_string()?); + // hsState 1 is hidden, 2 is veryHidden: omitted entirely, + // heading included, exactly as in xlsx. + if state == 1 || state == 2 { + continue; + } + match rid { + Some(rid) => sheets.push((name, rid)), + None => log::warn!("skipping sheet {name:?} with no worksheet relationship"), + } + } + _ => {} + } + } + Ok((date1904, sheets)) +} + +/// `styles.bin` reduced to the ordered cellXfs list, each entry's format id +/// resolved to a parsed format. Only BrtXF records between BrtBeginCellXFs +/// and BrtEndCellXFs are cell XFs; the cell-style XF collection also holds +/// BrtXF records and must not shift the indices. +fn read_styles(data: &[u8]) -> Result, ConvertError> { + let mut codes: HashMap = HashMap::new(); + let mut fmt_ids: Vec = Vec::new(); + let mut in_cell_xfs = false; + let mut records = Records::new(data); + while let Some((id, payload)) = records.next()? { + match id { + BRT_BEGIN_CELL_XFS => in_cell_xfs = true, + BRT_END_CELL_XFS => in_cell_xfs = false, + BRT_FMT => { + let mut f = Fields::new(payload); + let ifmt = f.u16()?; + codes.insert(u32::from(ifmt), f.wide_string()?); + } + BRT_XF if in_cell_xfs => { + let mut f = Fields::new(payload); + f.u16()?; // ixfeParent + fmt_ids.push(u32::from(f.u16()?)); + } + _ => {} + } + } + let custom: HashMap = codes.iter().map(|(k, v)| (*k, v.as_str())).collect(); + let mut cache: HashMap = HashMap::new(); + Ok(fmt_ids + .iter() + .map(|id| cache.entry(*id).or_insert_with(|| resolve_format(*id, &custom)).clone()) + .collect()) +} + +/// `sharedStrings.bin`: one cleaned entry per BrtSSTItem in order. The item +/// is a RichStr; formatting runs and phonetic data trail the string and are +/// not content. +fn read_shared_strings(data: &[u8]) -> Result, ConvertError> { + let mut out = Vec::new(); + let mut records = Records::new(data); + while let Some((id, payload)) = records.next()? { + if id == BRT_SST_ITEM { + let mut f = Fields::new(payload); + f.u8()?; // fRichStr / fExtStr flags + out.push(clean_text(&f.wide_string()?)); + } + } + Ok(out) +} + +/// One worksheet part. Rows come from the preceding BrtRowHdr; every cell +/// record carries only its column and cellXfs index in the common cell +/// header (MS-XLSB 2.5.10). +fn read_sheet( + data: &[u8], + shared: &[String], + xfs: &[CellFormat], + date1904: bool, +) -> Result { + let mut out = SheetContent::default(); + let mut row: Option = None; + let mut records = Records::new(data); + while let Some((id, payload)) = records.next()? { + let mut f = Fields::new(payload); + match id { + BRT_ROW_HDR => { + let rw = f.u32()?; + if rw >= MAX_ROWS { + row = None; + continue; + } + row = Some(rw); + f.skip(7)?; // ixfe, miyRw, fExtraAsc/fExtraDsc byte + if f.u8()? & 0x10 != 0 { + // fDyZero: the row is hidden. + out.hidden_rows.insert(rw); + } + } + BRT_COL_INFO => { + let first = f.u32()?; + let last = f.u32()?; + f.skip(8)?; // coldx, ixfe + let hidden = f.u16()? & 1 != 0; + if hidden && first <= last && first < MAX_COLS { + out.hidden_cols.push((first, last.min(MAX_COLS - 1))); + } + } + BRT_MERGE_CELL => { + let (r1, r2, c1, c2) = (f.u32()?, f.u32()?, f.u32()?, f.u32()?); + if r1.max(r2) >= MAX_ROWS || c1.max(c2) >= MAX_COLS { + log::debug!("skipping out-of-bounds merge region"); + continue; + } + if r1 != r2 || c1 != c2 { + out.merges.push((r1.min(r2), c1.min(c2), r1.max(r2), c1.max(c2))); + } + } + BRT_CELL_RK | BRT_CELL_ERROR | BRT_CELL_BOOL | BRT_CELL_REAL | BRT_CELL_ST + | BRT_CELL_ISST | BRT_CELL_RSTRING | BRT_FMLA_STRING | BRT_FMLA_NUM | BRT_FMLA_BOOL + | BRT_FMLA_ERROR => { + let col = f.u32()?; + let style = f.u32()? & 0x00FF_FFFF; + let Some(r) = row else { + log::debug!("skipping cell record before any row header"); + continue; + }; + if col >= MAX_COLS { + continue; + } + let fmt = xfs.get(style as usize).unwrap_or(&CellFormat::General); + let text = cell_text(id, &mut f, fmt, shared, date1904)?; + if !text.is_empty() { + out.cells.insert((r, col), text); + } + } + _ => {} + } + } + Ok(out) +} + +/// A cell's rendered text from the record body after the common header. +/// Formula records carry their cached result first; the parsed formula +/// trails it and is never read. +fn cell_text( + id: u16, + f: &mut Fields, + fmt: &CellFormat, + shared: &[String], + date1904: bool, +) -> Result { + Ok(match id { + BRT_CELL_RK => render_number(fmt, rk_number(f.u32()?), date1904), + BRT_CELL_REAL | BRT_FMLA_NUM => render_number(fmt, f.f64()?, date1904), + BRT_CELL_ST | BRT_FMLA_STRING => format_as_text(fmt, &clean_text(&f.wide_string()?)), + BRT_CELL_RSTRING => { + f.u8()?; // RichStr flags; runs and phonetic data trail the string + format_as_text(fmt, &clean_text(&f.wide_string()?)) + } + BRT_CELL_ISST => { + let isst = f.u32()?; + match usize::try_from(isst).ok().and_then(|i| shared.get(i)) { + Some(text) => format_as_text(fmt, text), + None => { + log::debug!("shared string index {isst} out of range"); + String::new() + } + } + } + BRT_CELL_BOOL | BRT_FMLA_BOOL => match f.u8()? { + 1 => "TRUE".to_string(), + 0 => "FALSE".to_string(), + _ => String::new(), + }, + BRT_CELL_ERROR | BRT_FMLA_ERROR => { + let code = f.u8()?; + match error_literal(code) { + Some(text) => text.to_string(), + None => { + log::debug!("unknown error cell code {code:#04x}"); + String::new() + } + } + } + _ => String::new(), + }) +} + +/// Iterator over one part's record stream: a 1-2 byte record type and a 1-4 +/// byte payload size, both 7 bits per byte with the high bit as the +/// continuation flag (MS-XLSB 2.1.4). +struct Records<'a> { + data: &'a [u8], + pos: usize, + seen: u64, +} + +impl<'a> Records<'a> { + fn new(data: &'a [u8]) -> Self { + Records { data, pos: 0, seen: 0 } + } + + fn next(&mut self) -> Result, ConvertError> { + if self.pos >= self.data.len() { + return Ok(None); + } + self.seen += 1; + if self.seen > limits::MAX_RECORDS { + return Err(ConvertError::ResourceLimit { + limit: "max_records", + detail: "record stream exceeds the record budget".into(), + }); + } + let b0 = self.byte()?; + let id = if b0 & 0x80 != 0 { + u16::from(b0 & 0x7F) | (u16::from(self.byte()? & 0x7F) << 7) + } else { + u16::from(b0) + }; + let mut size: usize = 0; + for i in 0..4 { + let b = self.byte()?; + size |= usize::from(b & 0x7F) << (7 * i); + if b & 0x80 == 0 { + break; + } + if i == 3 { + return Err(ConvertError::malformed("record size runs past four bytes")); + } + } + let end = self + .pos + .checked_add(size) + .filter(|&end| end <= self.data.len()) + .ok_or_else(|| ConvertError::malformed("record payload runs past the part"))?; + let payload = &self.data[self.pos..end]; + self.pos = end; + Ok(Some((id, payload))) + } + + fn byte(&mut self) -> Result { + let b = *self + .data + .get(self.pos) + .ok_or_else(|| ConvertError::malformed("truncated record header"))?; + self.pos += 1; + Ok(b) + } +} + +/// Bounds-checked field reads within one record payload. Every length here +/// is attacker-controlled, so nothing is read without a check. +struct Fields<'a> { + data: &'a [u8], + pos: usize, +} + +impl<'a> Fields<'a> { + fn new(data: &'a [u8]) -> Self { + Fields { data, pos: 0 } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8], ConvertError> { + let end = self + .pos + .checked_add(n) + .filter(|&end| end <= self.data.len()) + .ok_or_else(|| ConvertError::malformed("truncated record payload"))?; + let bytes = &self.data[self.pos..end]; + self.pos = end; + Ok(bytes) + } + + fn skip(&mut self, n: usize) -> Result<(), ConvertError> { + self.take(n).map(|_| ()) + } + + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } + + fn u16(&mut self) -> Result { + Ok(u16::from_le_bytes(self.take(2)?.try_into().expect("2-byte slice"))) + } + + fn u32(&mut self) -> Result { + Ok(u32::from_le_bytes(self.take(4)?.try_into().expect("4-byte slice"))) + } + + fn f64(&mut self) -> Result { + Ok(f64::from_le_bytes(self.take(8)?.try_into().expect("8-byte slice"))) + } + + /// XLWideString (MS-XLSB 2.5.169): a u32 character count, then that many + /// UTF-16LE code units. Unpaired surrogates decode to U+FFFD. + fn wide_string(&mut self) -> Result { + let cch = self.u32()? as usize; + let bytes = cch + .checked_mul(2) + .ok_or_else(|| ConvertError::malformed("oversized string length")) + .and_then(|n| self.take(n))?; + let units = bytes.chunks_exact(2).map(|pair| u16::from_le_bytes([pair[0], pair[1]])); + Ok(char::decode_utf16(units).map(|r| r.unwrap_or(char::REPLACEMENT_CHARACTER)).collect()) + } + + /// XLNullableWideString: the reserved count 0xFFFFFFFF is the null + /// string. + fn nullable_wide_string(&mut self) -> Result, ConvertError> { + if self.data.get(self.pos..self.pos + 4) == Some(&[0xFF; 4]) { + self.pos += 4; + return Ok(None); + } + self.wide_string().map(Some) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{CellSlot, Table, inlines_to_plain_text}; + use std::io::Write; + + const PKG_RELS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; + const WS_REL: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"; + const STYLES_REL: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles"; + + /// Route through the shared container dispatch, the only entry a caller + /// has. + fn parse(bytes: &[u8]) -> Result { + super::super::parse(bytes) + } + + /// One record in canonical (shortest) framing. + fn rec(id: u16, payload: &[u8]) -> Vec { + let mut out = Vec::new(); + if id < 0x80 { + out.push(id as u8); + } else { + out.push((id & 0x7F) as u8 | 0x80); + out.push((id >> 7) as u8); + } + let mut size = payload.len() as u32; + loop { + let low = (size & 0x7F) as u8; + size >>= 7; + if size == 0 { + out.push(low); + break; + } + out.push(low | 0x80); + } + out.extend_from_slice(payload); + out + } + + fn wstr(s: &str) -> Vec { + let units: Vec = s.encode_utf16().collect(); + let mut out = (units.len() as u32).to_le_bytes().to_vec(); + for u in units { + out.extend_from_slice(&u.to_le_bytes()); + } + out + } + + /// The 8-byte common cell header. + fn cell(col: u32, style: u32) -> Vec { + [col.to_le_bytes(), style.to_le_bytes()].concat() + } + + fn row_hdr(r: u32, hidden: bool) -> Vec { + let mut p = r.to_le_bytes().to_vec(); + p.extend_from_slice(&[0; 4]); // ixfe + p.extend_from_slice(&[0; 2]); // miyRw + p.push(0); // fExtraAsc / fExtraDsc + p.push(if hidden { 0x10 } else { 0 }); // iOutLevel..fDyZero.. + p.push(0); // fPhShow + p.extend_from_slice(&0u32.to_le_bytes()); // ccolspan + rec(BRT_ROW_HDR, &p) + } + + fn real_cell(col: u32, style: u32, v: f64) -> Vec { + let mut p = cell(col, style); + p.extend_from_slice(&v.to_le_bytes()); + rec(BRT_CELL_REAL, &p) + } + + fn col_info(first: u32, last: u32, hidden: bool) -> Vec { + let mut p = first.to_le_bytes().to_vec(); + p.extend_from_slice(&last.to_le_bytes()); + p.extend_from_slice(&[0; 8]); // coldx, ixfe + p.extend_from_slice(&u16::from(hidden).to_le_bytes()); + rec(BRT_COL_INFO, &p) + } + + fn merge(r1: u32, r2: u32, c1: u32, c2: u32) -> Vec { + let p = [r1.to_le_bytes(), r2.to_le_bytes(), c1.to_le_bytes(), c2.to_le_bytes()].concat(); + rec(BRT_MERGE_CELL, &p) + } + + /// styles.bin: custom BrtFmt entries and the cellXfs list by numFmtId. A + /// cell-style BrtXF precedes BrtBeginCellXFs to prove it shifts nothing. + fn styles(fmts: &[(u16, &str)], xf_fmt_ids: &[u16]) -> Vec { + let xf = |parent: u16, ifmt: u16| { + let mut p = parent.to_le_bytes().to_vec(); + p.extend_from_slice(&ifmt.to_le_bytes()); + p.extend_from_slice(&[0; 12]); + rec(BRT_XF, &p) + }; + let mut out = Vec::new(); + for (id, code) in fmts { + let mut p = id.to_le_bytes().to_vec(); + p.extend(wstr(code)); + out.extend(rec(BRT_FMT, &p)); + } + out.extend(xf(0xFFFF, 9)); // cell-style XF outside cellXfs + out.extend(rec(BRT_BEGIN_CELL_XFS, &(xf_fmt_ids.len() as u32).to_le_bytes())); + for id in xf_fmt_ids { + out.extend(xf(0, *id)); + } + out.extend(rec(BRT_END_CELL_XFS, &[])); + out + } + + fn shared_strings(items: &[&str]) -> Vec { + let mut out = Vec::new(); + for s in items { + let mut p = vec![0u8]; // no rich runs, no phonetic data + p.extend(wstr(s)); + out.extend(rec(BRT_SST_ITEM, &p)); + } + out + } + + /// Assemble a workbook: (name, hsState, sheet records) per sheet, plus + /// optional styles and shared-string parts. + #[derive(Default)] + struct Wb<'a> { + sheets: Vec<(&'a str, u32, Vec)>, + styles: Option>, + shared: Option>, + date1904: bool, + } + + impl Wb<'_> { + fn build(&self) -> Vec { + let mut workbook = Vec::new(); + let mut prop = u32::from(self.date1904).to_le_bytes().to_vec(); + prop.extend_from_slice(&0u32.to_le_bytes()); // dwThemeVersion + prop.extend(wstr("")); // strName + workbook.extend(rec(BRT_WB_PROP, &prop)); + let mut rels = String::new(); + for (i, (name, state, _)) in self.sheets.iter().enumerate() { + let id = i + 1; + let mut p = state.to_le_bytes().to_vec(); + p.extend_from_slice(&(id as u32).to_le_bytes()); // iTabID + p.extend(wstr(&format!("rId{id}"))); + p.extend(wstr(name)); + workbook.extend(rec(BRT_BUNDLE_SH, &p)); + rels.push_str(&format!( + r#""# + )); + } + if self.styles.is_some() { + rels.push_str(&format!( + r#""# + )); + } + if self.shared.is_some() { + rels.push_str(&format!( + r#""# + )); + } + let rels = format!( + r#"{rels}"# + ); + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + let mut add = |name: &str, body: &[u8]| { + zip.start_file(name, opts).unwrap(); + zip.write_all(body).unwrap(); + }; + add("xl/workbook.bin", &workbook); + add("xl/_rels/workbook.bin.rels", rels.as_bytes()); + for (i, (_, _, body)) in self.sheets.iter().enumerate() { + add(&format!("xl/worksheets/sheet{}.bin", i + 1), body); + } + if let Some(styles) = &self.styles { + add("xl/styles.bin", styles); + } + if let Some(shared) = &self.shared { + add("xl/sharedStrings.bin", shared); + } + zip.finish().unwrap().into_inner() + } + } + + fn one_sheet(body: Vec) -> Wb<'static> { + Wb { sheets: vec![("S", 0, body)], ..Wb::default() } + } + + fn first_table(doc: &Document) -> &Table { + match doc.blocks.iter().find_map(|b| match b { + Block::Table(t) => Some(t), + _ => None, + }) { + Some(t) => t, + None => panic!("expected a table, got {:?}", doc.blocks), + } + } + + fn texts(table: &Table) -> Vec> { + table + .grid + .iter() + .map(|row| { + row.iter() + .map(|slot| match slot { + CellSlot::Origin(cell) => cell + .blocks + .iter() + .filter_map(|b| match b { + Block::Paragraph(i) => Some(inlines_to_plain_text(i)), + _ => None, + }) + .collect(), + CellSlot::Covered { .. } => "".to_string(), + }) + .collect() + }) + .collect() + } + + #[test] + fn record_framing_uses_seven_bit_groups() { + // A record id over 0x7F takes two bytes, a payload size over 0x7F + // takes more than one; the exact encodings are pinned so a framing + // bug cannot hide behind a matching writer bug. + let mut stream = rec(BRT_BUNDLE_SH, &[7u8; 300]); + assert_eq!(&stream[..4], &[0x9C, 0x01, 0xAC, 0x02]); + stream.extend(rec(BRT_CELL_RK, &[1, 2, 3])); + let mut records = Records::new(&stream); + let (id, payload) = records.next().unwrap().unwrap(); + assert_eq!((id, payload.len()), (BRT_BUNDLE_SH, 300)); + let (id, payload) = records.next().unwrap().unwrap(); + assert_eq!((id, payload), (BRT_CELL_RK, &[1u8, 2, 3][..])); + assert!(records.next().unwrap().is_none()); + } + + #[test] + fn truncated_record_stream_is_malformed() { + let mut stream = rec(BRT_CELL_RK, &[0u8; 10]); + stream.truncate(stream.len() - 1); + let mut records = Records::new(&stream); + assert!(records.next().is_err(), "payload shorter than its declared size must error"); + } + + #[test] + fn rk_values_decode_both_kinds_with_and_without_x100() { + // Integer, negative integer (arithmetic shift), integer / 100, + // float from the high 30 bits, float / 100. + assert_eq!(rk_number((1 << 2) | 2), 1.0); + assert_eq!(rk_number(((-1i32 as u32) << 2) | 2), -1.0); + assert_eq!(rk_number((12345 << 2) | 2 | 1), 123.45); + assert_eq!(rk_number((1.5f64.to_bits() >> 32) as u32), 1.5); + assert_eq!(rk_number((1.5f64.to_bits() >> 32) as u32 | 1), 0.015); + } + + #[test] + fn number_formats_apply_to_stored_values() { + // Same expectations as the xlsx reader: percent and currency render + // their display value, a date format keeps the ISO rendering, and + // the cell-style XF in styles.bin does not shift cellXfs indices. + let mut body = row_hdr(0, false); + body.extend({ + let mut p = cell(0, 1); + p.extend_from_slice(&(((75 << 2) | 2 | 1) as u32).to_le_bytes()); // RK 0.75 via x100 + rec(BRT_CELL_RK, &p) + }); + body.extend(real_cell(1, 2, 1234.5)); + body.extend(real_cell(2, 3, 46096.0)); + body.extend(real_cell(3, 0, 1234.5)); // General + let wb = Wb { + styles: Some(styles( + &[(164, "0.0%"), (165, "\"$\"#,##0.00"), (166, "mm/dd/yyyy")], + &[0, 164, 165, 166], + )), + ..one_sheet(body) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!( + texts(first_table(&doc)), + vec![vec!["75.0%", "$1,234.50", "2026-03-15", "1234.5"]] + ); + } + + #[test] + fn strings_resolve_from_the_shared_table_and_inline() { + let mut body = row_hdr(0, false); + body.extend({ + let mut p = cell(0, 0); + p.extend_from_slice(&1u32.to_le_bytes()); + rec(BRT_CELL_ISST, &p) + }); + body.extend({ + let mut p = cell(1, 0); + p.extend(wstr(" inline ")); + rec(BRT_CELL_ST, &p) + }); + let wb = Wb { shared: Some(shared_strings(&["zeroth", "shared"])), ..one_sheet(body) }; + let doc = parse(&wb.build()).unwrap(); + // Untrimmed: leading/trailing whitespace in a cell is source content. + assert_eq!(texts(first_table(&doc)), vec![vec!["shared", " inline "]]); + } + + #[test] + fn bool_and_error_cells_render_literally() { + let mut body = row_hdr(0, false); + let byte_cell = |id, col, v: u8| { + let mut p = cell(col, 0); + p.push(v); + rec(id, &p) + }; + body.extend(byte_cell(BRT_CELL_BOOL, 0, 1)); + body.extend(byte_cell(BRT_CELL_BOOL, 1, 0)); + body.extend(byte_cell(BRT_CELL_ERROR, 2, 0x07)); + body.extend(byte_cell(BRT_CELL_ERROR, 3, 0x2A)); + let doc = parse(&one_sheet(body).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["TRUE", "FALSE", "#DIV/0!", "#N/A"]]); + } + + #[test] + fn formula_records_render_their_cached_result() { + // The trailing GrbitFmla and CellParsedFormula bytes are payload the + // reader must skip untouched. + let mut body = row_hdr(0, false); + body.extend({ + let mut p = cell(0, 0); + p.extend_from_slice(&2.5f64.to_le_bytes()); + p.extend_from_slice(&[0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x1E, 0x05, 0x00]); + rec(BRT_FMLA_NUM, &p) + }); + body.extend({ + let mut p = cell(1, 0); + p.extend(wstr("computed")); + p.extend_from_slice(&[0x00, 0x00, 0x00, 0x00, 0x00, 0x00]); + rec(BRT_FMLA_STRING, &p) + }); + let doc = parse(&one_sheet(body).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["2.5", "computed"]]); + } + + #[test] + fn hidden_rows_columns_and_sheets_are_omitted() { + let mut visible = col_info(1, 1, true); + visible.extend(row_hdr(0, false)); + visible.extend(real_cell(0, 0, 1.0)); + visible.extend(real_cell(1, 0, 2.0)); // hidden column + visible.extend(real_cell(2, 0, 3.0)); + visible.extend(row_hdr(1, true)); + visible.extend(real_cell(0, 0, 4.0)); // hidden row + visible.extend(row_hdr(2, false)); + visible.extend(real_cell(0, 0, 5.0)); + let mut secret = row_hdr(0, false); + secret.extend(real_cell(0, 0, 9.0)); + let wb = Wb { + sheets: vec![ + ("Shown", 0, visible), + ("Hidden", 1, secret.clone()), + ("VeryHidden", 2, secret), + ], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(doc.blocks.len(), 1, "hidden sheets must add no heading and no table"); + assert_eq!(texts(first_table(&doc)), vec![vec!["1", "3"], vec!["5", ""]]); + } + + #[test] + fn merge_extends_past_the_populated_range() { + // Issue #8, matching the xlsx reader: the only populated cell + // anchors F1:O3, so the grid widens to the full 3x10 extent. + let mut body = row_hdr(0, false); + body.extend({ + let mut p = cell(5, 0); + p.extend(wstr("wide")); + rec(BRT_CELL_ST, &p) + }); + body.extend(merge(0, 2, 5, 14)); + let doc = parse(&one_sheet(body).build()).unwrap(); + let table = first_table(&doc); + assert_eq!((table.grid.len(), table.grid[1].len()), (3, 10)); + let CellSlot::Origin(cell) = &table.grid[0][0] else { + panic!("expected the merge origin at (0,0)"); + }; + assert_eq!((cell.col_span, cell.row_span), (10, 3)); + } + + #[test] + fn date1904_serials_shift_epoch() { + let mut body = row_hdr(0, false); + body.extend(real_cell(0, 0, 100.0)); + let wb = Wb { + styles: Some(styles(&[(164, "yyyy-mm-dd")], &[164])), + date1904: true, + ..one_sheet(body) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["1904-04-10"]]); + } + + #[test] + fn unknown_records_are_skipped_without_desync() { + // A record the reader does not know (BrtWsDim, 148) must be skipped + // by its declared size, leaving the following cells intact. + let mut body = rec(148, &[0u8; 16]); + body.extend(row_hdr(0, false)); + body.extend(real_cell(0, 0, 7.0)); + let doc = parse(&one_sheet(body).build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["7"]]); + } +} diff --git a/src/formats/sheet/xlsx.rs b/src/formats/sheet/xlsx.rs new file mode 100644 index 00000000..f4efbc9b --- /dev/null +++ b/src/formats/sheet/xlsx.rs @@ -0,0 +1,1016 @@ +//! In-house SpreadsheetML reader (.xlsx / .xlsm): the workbook's visible +//! sheets, shared strings, cell number formats from `xl/styles.xml`, and +//! merge regions. Rows, columns, and sheets the source hides are omitted, +//! and merge regions are remapped onto the surviving grid. + +use super::numfmt::{DateParts, NumberFormat, Rendered, builtin_code}; +use super::{format_duration_days, format_float, format_time_of_day}; +use crate::error::ConvertError; +use crate::model::{Block, Cell, Document, GridBuilder, Inline, Table, TableKind}; +use crate::package::limits; +use crate::package::relationships::{Relationships, read_rels, rel_type, rels_part_for}; +use crate::package::xml::{Element, ns}; +use crate::package::{Package, path}; +use crate::shared::header::resolve_header_rows; +use crate::shared::text::clean_text; +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +pub(super) const SHARED_STRINGS_REL: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/sharedStrings"; + +/// The grid bounds the format defines; a reference outside them is not a +/// real cell. +pub(super) const MAX_ROWS: u32 = 1_048_576; +pub(super) const MAX_COLS: u32 = 16_384; + +pub(super) fn parse(pkg: &mut Package, wb_part: &str) -> Result { + let workbook = pkg.required_xml_part(wb_part)?; + // Any OOXML main part resolves here; a document or presentation would + // otherwise convert to an empty workbook. + if !workbook.child_elems().next().is_some_and(|e| e.is(ns::SML, "workbook")) { + return Err(ConvertError::malformed("main part is not a workbook")); + } + let wb_rels = read_rels(pkg, &rels_part_for(wb_part))?; + let date1904 = workbook + .first_descendant(ns::SML, "workbookPr") + .and_then(|e| e.attr_unqualified("date1904")) + .is_some_and(bool_attr); + + let shared = + match sibling_part(pkg, &wb_rels, wb_part, SHARED_STRINGS_REL, "sharedStrings.xml")? { + Some(root) => shared_strings(&root), + None => Vec::new(), + }; + let styles = + Styles::read(sibling_part(pkg, &wb_rels, wb_part, rel_type::STYLES, "styles.xml")?); + + // Visible sheets in workbook order; hidden and veryHidden sheets are + // omitted entirely, heading included. + let mut sheets: Vec<(String, String)> = Vec::new(); + for sheet in workbook + .first_descendant(ns::SML, "sheets") + .into_iter() + .flat_map(|s| s.find_all(ns::SML, "sheet")) + { + if matches!(sheet.attr_unqualified("state"), Some("hidden" | "veryHidden")) { + continue; + } + let name = sheet.attr_unqualified("name").unwrap_or_default().to_string(); + let Some(target) = + sheet.attr_qualified(ns::R, "id").and_then(|rid| wb_rels.internal_target(rid)) + else { + log::warn!("skipping sheet {name:?} with no worksheet relationship"); + continue; + }; + match path::resolve(wb_part, target) { + Ok(t) => sheets.push((name, t.path)), + Err(e) => log::warn!("skipping sheet {name:?} with unresolvable target: {e}"), + } + } + + let multi_sheet = sheets.len() > 1; + let mut doc = Document::default(); + let mut failed = 0usize; + // One budget for the workbook, so sheets cannot multiply the cap. + let mut slots = 0u64; + for (name, part) in &sheets { + let worksheet = pkg.optional_xml_part(part)?; + let Some(worksheet) = worksheet.as_ref().and_then(|r| r.find(ns::SML, "worksheet")) else { + log::warn!("skipping unreadable sheet {name:?}"); + failed += 1; + continue; + }; + let content = read_sheet(worksheet, &shared, &styles, date1904); + let Some(table) = build_table(content, &mut slots)? else { + continue; + }; + if multi_sheet { + doc.blocks.push(Block::heading(2, vec![Inline::plain(name.clone())])); + } + doc.blocks.push(Block::Table(table)); + } + if !sheets.is_empty() && failed == sheets.len() { + return Err(ConvertError::malformed("no sheet in the workbook could be read")); + } + Ok(doc) +} + +/// Part name for a workbook-level sibling: the relationship of the given +/// type when present, else the conventional name next to the workbook part. +pub(super) fn sibling_part_name( + rels: &Relationships, + base: &str, + rel: &str, + conventional: &str, +) -> String { + rels.first_of_type(rel) + .and_then(|r| path::resolve(base, &r.target).ok()) + .map(|t| t.path) + .unwrap_or_else(|| match base.rsplit_once('/') { + Some((dir, _)) => format!("{dir}/{conventional}"), + None => conventional.to_string(), + }) +} + +/// Load a workbook-level XML part by relationship type, falling back to the +/// conventional name next to the workbook part. +fn sibling_part( + pkg: &mut Package, + rels: &Relationships, + base: &str, + rel: &str, + conventional: &str, +) -> Result, ConvertError> { + pkg.optional_xml_part(&sibling_part_name(rels, base, rel, conventional)) +} + +/// The shared string table, one cleaned entry per `si` in order. +fn shared_strings(root: &Element) -> Vec { + let Some(sst) = root.find(ns::SML, "sst") else { + return Vec::new(); + }; + sst.find_all(ns::SML, "si").map(|si| clean_text(&rich_text(si))).collect() +} + +/// Text of an `si` or `is`: a single `t`, or rich-text `r` runs +/// concatenated. Phonetic guides (`rPh`) are not content. +fn rich_text(item: &Element) -> String { + let mut out = String::new(); + for child in item.child_elems() { + if child.is(ns::SML, "t") { + out.push_str(&child.text()); + } else if child.is(ns::SML, "r") + && let Some(t) = child.find(ns::SML, "t") + { + out.push_str(&t.text()); + } + } + out +} + +/// A cell's resolved number format: General, or a parsed format code. +/// Shared with the BIFF reader - .xls XF/FORMAT records resolve into the +/// same representation. +#[derive(Clone)] +pub(super) enum CellFormat { + General, + Fmt(Rc), +} + +/// `xl/styles.xml` reduced to what rendering needs: the ordered `cellXfs` +/// list, each entry's numFmtId resolved to a parsed format. +struct Styles { + xfs: Vec, +} + +impl Styles { + fn read(root: Option) -> Styles { + let Some(root) = root else { + return Styles { xfs: Vec::new() }; + }; + let mut custom: HashMap = HashMap::new(); + for fmts in root.descendants(ns::SML, "numFmts") { + for nf in fmts.find_all(ns::SML, "numFmt") { + if let (Some(id), Some(code)) = ( + nf.attr_unqualified("numFmtId").and_then(|v| v.parse().ok()), + nf.attr_unqualified("formatCode"), + ) { + custom.insert(id, code); + } + } + } + let mut cache: HashMap = HashMap::new(); + let xfs = root + .first_descendant(ns::SML, "cellXfs") + .map(|xfs| { + xfs.find_all(ns::SML, "xf") + .map(|xf| { + let id = xf + .attr_unqualified("numFmtId") + .and_then(|v| v.parse().ok()) + .unwrap_or(0u32); + cache.entry(id).or_insert_with(|| resolve_format(id, &custom)).clone() + }) + .collect() + }) + .unwrap_or_default(); + Styles { xfs } + } + + /// The format for a cell's `s` attribute, an index into `cellXfs` + /// (default 0). + fn for_cell(&self, s: Option<&str>) -> &CellFormat { + let i = s.and_then(|s| s.parse::().ok()).unwrap_or(0); + self.xfs.get(i).unwrap_or(&CellFormat::General) + } +} + +/// A numFmtId's format: the file's own `numFmt` entries first, then the +/// built-in table. Unknown ids and unsupported codes fall back to General - +/// never to a guess. +pub(super) fn resolve_format(id: u32, custom: &HashMap) -> CellFormat { + let code = custom.get(&id).copied().or_else(|| builtin_code(id)); + match code { + Some(code) => match NumberFormat::parse(code) { + Some(f) => CellFormat::Fmt(Rc::new(f)), + None => { + log::debug!("unsupported number format {code:?}, rendering as General"); + CellFormat::General + } + }, + None => { + if id != 0 { + log::debug!("numFmtId {id} has no resolvable code, rendering as General"); + } + CellFormat::General + } + } +} + +/// One worksheet, parsed but not yet filtered or gridded. Shared with the +/// BIFF reader, which fills it from records instead of XML. +#[derive(Default)] +pub(super) struct SheetContent { + /// Rendered text by zero-based (row, col); empty results are absent. + pub(super) cells: HashMap<(u32, u32), String>, + pub(super) hidden_rows: HashSet, + /// Inclusive zero-based column ranges hidden by `cols/col` entries. + pub(super) hidden_cols: Vec<(u32, u32)>, + /// Inclusive zero-based merge regions (r1, c1, r2, c2), area > 1. + pub(super) merges: Vec<(u32, u32, u32, u32)>, +} + +fn read_sheet( + worksheet: &Element, + shared: &[String], + styles: &Styles, + date1904: bool, +) -> SheetContent { + let mut out = SheetContent::default(); + for cols in worksheet.find_all(ns::SML, "cols") { + for col in cols.find_all(ns::SML, "col") { + if !col.attr_unqualified("hidden").is_some_and(bool_attr) { + continue; + } + let bound = |name| { + col.attr_unqualified(name) + .and_then(|v| v.parse::().ok()) + .and_then(|v| v.checked_sub(1)) + }; + if let (Some(min), Some(max)) = (bound("min"), bound("max")) + && min <= max + { + out.hidden_cols.push((min, max.min(MAX_COLS - 1))); + } + } + } + let mut next_row: u32 = 0; + for row in worksheet.find_all(ns::SML, "sheetData").flat_map(|sd| sd.find_all(ns::SML, "row")) { + let r = row + .attr_unqualified("r") + .and_then(|v| v.parse::().ok()) + .and_then(|v| v.checked_sub(1)) + .unwrap_or(next_row); + if r >= MAX_ROWS { + continue; + } + next_row = r + 1; + if row.attr_unqualified("hidden").is_some_and(bool_attr) { + out.hidden_rows.insert(r); + } + let mut next_col: u32 = 0; + for c in row.find_all(ns::SML, "c") { + // Position comes from the cell's own reference: a row may skip + // cells entirely, so iteration order says nothing. + let (cr, cc) = match c.attr_unqualified("r").map(parse_ref) { + Some(Some(rc)) => rc, + Some(None) => continue, + None => (r, next_col), + }; + next_col = cc + 1; + if cr >= MAX_ROWS || cc >= MAX_COLS { + continue; + } + let text = cell_text(c, shared, styles, date1904); + if !text.is_empty() { + out.cells.insert((cr, cc), text); + } + } + } + for merge in + worksheet.find_all(ns::SML, "mergeCells").flat_map(|mc| mc.find_all(ns::SML, "mergeCell")) + { + let Some(region) = merge.attr_unqualified("ref").and_then(parse_region) else { + log::debug!("skipping unparseable merge reference"); + continue; + }; + let (r1, c1, r2, c2) = region; + if r1 != r2 || c1 != c2 { + out.merges.push(region); + } + } + out +} + +/// A cell's rendered text, per its `t` type and resolved number format. +fn cell_text(c: &Element, shared: &[String], styles: &Styles, date1904: bool) -> String { + let fmt = styles.for_cell(c.attr_unqualified("s")); + let value = || c.find(ns::SML, "v").map(|v| v.text()).unwrap_or_default(); + match c.attr_unqualified("t").unwrap_or("n") { + "s" => { + let v = value(); + match v.trim().parse::().ok().and_then(|i| shared.get(i)) { + Some(text) => format_as_text(fmt, text), + None => { + log::debug!("shared string index {v:?} out of range"); + String::new() + } + } + } + "str" => format_as_text(fmt, &clean_text(&value())), + "inlineStr" => { + let text = c.find(ns::SML, "is").map(|is| clean_text(&rich_text(is))); + format_as_text(fmt, &text.unwrap_or_default()) + } + "b" => match value().trim() { + "1" | "true" => "TRUE".to_string(), + "0" | "false" => "FALSE".to_string(), + _ => String::new(), + }, + "e" | "d" => clean_text(&value()), + _ => { + let v = value(); + let v = v.trim(); + if v.is_empty() { + return String::new(); + } + let Ok(n) = v.parse::() else { + log::debug!("unparseable numeric cell value {v:?}"); + return String::new(); + }; + render_number(fmt, n, date1904) + } + } +} + +/// Render a numeric cell through its resolved format. Shared with the BIFF +/// and binary readers so every container formats identically. +pub(super) fn render_number(fmt: &CellFormat, n: f64, date1904: bool) -> String { + let text = match fmt { + CellFormat::General => format_float(n), + CellFormat::Fmt(f) => match f.format_number(n) { + Rendered::General { value, prefix, suffix } => { + format!("{prefix}{}{suffix}", format_float(value)) + } + Rendered::Text(s) => s, + Rendered::DateTime(parts) => render_serial(n, parts, date1904), + }, + }; + clean_text(&text) +} + +pub(super) fn format_as_text(fmt: &CellFormat, text: &str) -> String { + match fmt { + CellFormat::Fmt(f) => match f.format_text(text) { + Some(s) => clean_text(&s), + None => text.to_string(), + }, + CellFormat::General => text.to_string(), + } +} + +/// Materialize a sheet's grid: visibility filtering, the populated extent +/// widened to cover intersecting merge regions (a merge anchored on the +/// only populated cell must survive at full size), and merges remapped onto +/// the surviving rows and columns. +pub(super) fn build_table( + mut sheet: SheetContent, + slots: &mut u64, +) -> Result, ConvertError> { + // Hidden coordinates as sorted lists: lookups and first-visible scans + // stay logarithmic, so an adversarial pile of hidden rows or column + // ranges cannot force quadratic work. + let hidden_rows = { + let mut rows: Vec = sheet.hidden_rows.iter().copied().collect(); + rows.sort_unstable(); + rows + }; + let hidden_cols = expand_ranges(&mut sheet.hidden_cols); + let hidden_row = |r: u32| hidden_rows.binary_search(&r).is_ok(); + let hidden_col = |c: u32| hidden_cols.binary_search(&c).is_ok(); + + // A merge with no surviving row or column disappears with its content. + // One whose origin is hidden keeps its content at the first surviving + // position it covers, so the value is not lost. + let cells = &mut sheet.cells; + sheet.merges.retain(|&(r1, c1, r2, c2)| { + let vr = first_visible(&hidden_rows, r1, r2); + let vc = first_visible(&hidden_cols, c1, c2); + let (Some(vr), Some(vc)) = (vr, vc) else { + return false; + }; + if (vr, vc) != (r1, c1) + && let Some(text) = cells.remove(&(r1, c1)) + { + cells.insert((vr, vc), text); + } + true + }); + + // Populated extent over visible cells only. + let mut bounds: Option<(u32, u32, u32, u32)> = None; + for &(r, c) in sheet.cells.keys() { + if hidden_row(r) || hidden_col(c) { + continue; + } + bounds = Some(match bounds { + None => (r, c, r, c), + Some((r1, c1, r2, c2)) => (r1.min(r), c1.min(c), r2.max(r), c2.max(c)), + }); + } + let Some((mut r1, mut c1, mut r2, mut c2)) = bounds else { + return Ok(None); + }; + // Merge regions touching the populated extent widen it to their full + // size; the rest are dropped, so a crafted merge list can neither force + // unbounded materialization nor saturate onto (0,0). + sheet.merges.retain(|&(mr1, mc1, mr2, mc2)| mr1 <= r2 && mr2 >= r1 && mc1 <= c2 && mc2 >= c1); + for &(mr1, mc1, mr2, mc2) in &sheet.merges { + (r1, c1, r2, c2) = (r1.min(mr1), c1.min(mc1), r2.max(mr2), c2.max(mc2)); + } + + let row_map: Vec = (r1..=r2).filter(|&r| !hidden_row(r)).collect(); + let col_map: Vec = (c1..=c2).filter(|&c| !hidden_col(c)).collect(); + if row_map.is_empty() || col_map.is_empty() { + return Ok(None); + } + // Charged before materializing, and across the workbook rather than per + // sheet: the extent comes from cell coordinates, so a handful of cells + // describes a whole sheet and a handful of sheets multiplies it. + *slots = slots.saturating_add(row_map.len() as u64 * col_map.len() as u64); + if *slots > limits::MAX_GRID_SLOTS { + return Err(ConvertError::ResourceLimit { + limit: "max_grid_slots", + detail: format!("workbook extent covers {slots} grid positions"), + }); + } + + // Remap merges onto the surviving coordinates. The covered-position set + // is charged against the expansion budget up front, before any + // insertion work, mirroring what placement would charge. + let visible_span = |map: &[u32], lo: u32, hi: u32| { + let a = map.partition_point(|&x| x < lo); + let b = map.partition_point(|&x| x <= hi); + (a, b - a) + }; + let mut origins: HashMap<(usize, usize), (u32, u32)> = HashMap::new(); + let mut covered: HashSet<(usize, usize)> = HashSet::new(); + let mut expansion = 0u64; + for &(mr1, mc1, mr2, mc2) in &sheet.merges { + let (r0, rn) = visible_span(&row_map, mr1, mr2); + let (c0, cn) = visible_span(&col_map, mc1, mc2); + if rn * cn <= 1 { + continue; + } + expansion = expansion.saturating_add((rn as u64) * (cn as u64) - 1); + if expansion > limits::MAX_EXPANSION { + return Err(ConvertError::ResourceLimit { + limit: "max_expansion", + detail: "merge region expansion exceeds the content budget".into(), + }); + } + origins.insert((r0, c0), (cn as u32, rn as u32)); + for r in r0..r0 + rn { + for c in c0..c0 + cn { + if (r, c) != (r0, c0) { + covered.insert((r, c)); + } + } + } + } + + let mut builder = GridBuilder::new(); + // A merge is real extent: trailing rows it covers stay in the grid. + builder.keep_covered_tail(); + for (ri, &row) in row_map.iter().enumerate() { + builder.next_row(); + for (ci, &col) in col_map.iter().enumerate() { + if covered.contains(&(ri, ci)) { + builder.covered(); + continue; + } + let cell = match sheet.cells.remove(&(row, col)) { + Some(text) => Cell::from_inlines(vec![Inline::plain(text)]), + None => Cell::default(), + }; + match origins.get(&(ri, ci)) { + Some(&(col_span, row_span)) => { + builder.place(Cell::spanning(cell.blocks, col_span, row_span))? + } + None => builder.place(cell)?, + } + } + } + // A spreadsheet marks no header row, so the shape of the data decides. + let mut table = builder.finish(TableKind::Data); + if table.grid.is_empty() { + return Ok(None); + } + table.header_rows = resolve_header_rows(&table, 0); + Ok(Some(table)) +} + +/// Flatten inclusive ranges into a sorted, deduplicated coordinate list. +/// Coalescing before expansion bounds the output by the coordinate space, +/// not by the range count. +fn expand_ranges(ranges: &mut [(u32, u32)]) -> Vec { + ranges.sort_unstable(); + let mut out = Vec::new(); + let mut next = 0u32; + for &(a, b) in ranges.iter() { + out.extend(a.max(next)..=b); + next = next.max(b.saturating_add(1)); + } + out +} + +/// First coordinate in `lo..=hi` absent from the sorted hidden list. The +/// consecutive hidden run starting at `lo` is measured by binary search, so +/// a long run cannot force a linear scan per query. +fn first_visible(hidden: &[u32], lo: u32, hi: u32) -> Option { + let tail = &hidden[hidden.partition_point(|&h| h < lo)..]; + // `tail[j] - j` never decreases, so "the run still holds at j" is a + // prefix property. + let (mut a, mut b) = (0usize, tail.len()); + while a < b { + let mid = (a + b) / 2; + if tail[mid] == lo + mid as u32 { + a = mid + 1; + } else { + b = mid; + } + } + let first = lo.checked_add(u32::try_from(a).ok()?)?; + (first <= hi).then_some(first) +} + +/// Render a date/time serial the way the crate always has: elapsed formats +/// as a duration, sub-day serials as a time of day, everything else as an +/// ISO-like date with the midnight time omitted. +fn render_serial(serial: f64, parts: DateParts, date1904: bool) -> String { + if !serial.is_finite() { + return format_float(serial); + } + if parts.elapsed { + return format_duration_days(serial); + } + if !parts.date { + return format_time_of_day(serial.fract()); + } + // A serial carrying no whole day has no date for a date format to show; + // the clock a combined format also names still renders. + if serial.abs() < 1.0 { + return if parts.time { format_time_of_day(serial) } else { format_float(serial) }; + } + // Out of the representable date range (through 9999-12-31): the serial + // is not a date, show the number. + if !(0.0..2_958_466.0).contains(&serial) { + return format_float(serial); + } + let mut days = serial.trunc() as i64; + // The fictitious 1900-02-29 has no date to render and would otherwise + // collapse onto serial 59. Tested before the seconds carry, so a value + // late on serial 59 still resolves as the date it belongs to. + if !date1904 && days == 60 { + return format_float(serial); + } + let mut secs = (serial.fract() * 86_400.0).round() as i64; + if secs >= 86_400 { + secs = 0; + days += 1; + } + let civil_days = if date1904 { + days + days_from_civil(1904, 1, 1) + } else { + // 1900 system: serial 1 is 1900-01-01, and the fictitious + // 1900-02-29 (serial 60) offsets everything after it by one day. + days - i64::from(days >= 60) + days_from_civil(1899, 12, 31) + }; + let (y, m, d) = civil_from_days(civil_days); + if !(1..=9999).contains(&y) { + return format_float(serial); + } + let mut out = format!("{y:04}-{m:02}-{d:02}"); + if parts.time && secs != 0 { + out.push_str(&format!(" {:02}:{:02}:{:02}", secs / 3600, (secs % 3600) / 60, secs % 60)); + } + out +} + +/// Days from 1970-01-01 to a civil date (Howard Hinnant's algorithm). +fn days_from_civil(y: i64, m: u32, d: u32) -> i64 { + let y = y - i64::from(m <= 2); + let era = y.div_euclid(400); + let yoe = y - era * 400; + let mp = i64::from((m + 9) % 12); + let doy = (153 * mp + 2) / 5 + i64::from(d) - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +/// A civil date from days since 1970-01-01 (Howard Hinnant's algorithm). +fn civil_from_days(z: i64) -> (i64, u32, u32) { + let z = z + 719_468; + let era = z.div_euclid(146_097); + let doe = z - era * 146_097; + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + (y + i64::from(m <= 2), m, d) +} + +/// A cell reference (`C3`) as zero-based (row, column); the column letters +/// are bijective base-26. +fn parse_ref(r: &str) -> Option<(u32, u32)> { + let digits_at = r.find(|c: char| c.is_ascii_digit())?; + let (letters, digits) = r.split_at(digits_at); + if letters.is_empty() { + return None; + } + let mut col: u32 = 0; + for ch in letters.chars() { + if !ch.is_ascii_alphabetic() { + return None; + } + col = col.checked_mul(26)?.checked_add(ch.to_ascii_uppercase() as u32 - 'A' as u32 + 1)?; + if col > MAX_COLS { + return None; + } + } + let row: u32 = digits.parse().ok()?; + if !(1..=MAX_ROWS).contains(&row) { + return None; + } + Some((row - 1, col - 1)) +} + +/// A merge reference (`F1:O3`, or a single cell) as an inclusive normalized +/// region. +fn parse_region(r: &str) -> Option<(u32, u32, u32, u32)> { + let (a, b) = r.split_once(':').unwrap_or((r, r)); + let (r1, c1) = parse_ref(a.trim())?; + let (r2, c2) = parse_ref(b.trim())?; + Some((r1.min(r2), c1.min(c2), r1.max(r2), c1.max(c2))) +} + +/// XML schema boolean attribute. +fn bool_attr(v: &str) -> bool { + matches!(v.trim(), "1" | "true") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{CellSlot, inlines_to_plain_text}; + use std::io::Write; + + const SML: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + const R: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + const PKG_RELS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; + const WS_REL: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet"; + const STYLES_REL: &str = + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles"; + + /// Route through the shared container dispatch, the only entry a caller + /// has. + fn parse(bytes: &[u8]) -> Result { + super::super::parse(bytes) + } + + /// Assemble a workbook: (name, state, worksheet body) per sheet, plus + /// optional styleSheet and sst parts. + #[derive(Default)] + struct Wb<'a> { + sheets: Vec<(&'a str, &'a str, &'a str)>, + styles: Option<&'a str>, + shared: Option<&'a str>, + date1904: bool, + } + + impl Wb<'_> { + fn build(&self) -> Vec { + let mut sheets = String::new(); + let mut rels = String::new(); + for (i, (name, state, _)) in self.sheets.iter().enumerate() { + let id = i + 1; + let state = + if state.is_empty() { String::new() } else { format!(" state=\"{state}\"") }; + sheets.push_str(&format!( + r#""# + )); + rels.push_str(&format!( + r#""# + )); + } + if self.styles.is_some() { + rels.push_str(&format!( + r#""# + )); + } + if self.shared.is_some() { + rels.push_str(&format!( + r#""# + )); + } + let pr = if self.date1904 { r#""# } else { "" }; + let workbook = format!( + r#"{pr}{sheets}"# + ); + let rels = format!( + r#"{rels}"# + ); + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + let mut add = |name: &str, body: &str| { + zip.start_file(name, opts).unwrap(); + zip.write_all(body.as_bytes()).unwrap(); + }; + add("xl/workbook.xml", &workbook); + add("xl/_rels/workbook.xml.rels", &rels); + for (i, (_, _, body)) in self.sheets.iter().enumerate() { + add( + &format!("xl/worksheets/sheet{}.xml", i + 1), + &format!(r#"{body}"#), + ); + } + if let Some(styles) = self.styles { + add( + "xl/styles.xml", + &format!( + r#"{styles}"# + ), + ); + } + if let Some(shared) = self.shared { + add( + "xl/sharedStrings.xml", + &format!(r#"{shared}"#), + ); + } + zip.finish().unwrap().into_inner() + } + } + + fn one_sheet(body: &str) -> Wb<'_> { + Wb { sheets: vec![("S", "", body)], ..Wb::default() } + } + + fn first_table(doc: &Document) -> &Table { + match doc.blocks.iter().find_map(|b| match b { + Block::Table(t) => Some(t), + _ => None, + }) { + Some(t) => t, + None => panic!("expected a table, got {:?}", doc.blocks), + } + } + + fn texts(table: &Table) -> Vec> { + table + .grid + .iter() + .map(|row| { + row.iter() + .map(|slot| match slot { + CellSlot::Origin(cell) => cell + .blocks + .iter() + .filter_map(|b| match b { + Block::Paragraph(i) => Some(inlines_to_plain_text(i)), + _ => None, + }) + .collect(), + CellSlot::Covered { .. } => "".to_string(), + }) + .collect() + }) + .collect() + } + + fn covered_count(table: &Table) -> usize { + table.grid.iter().flatten().filter(|s| matches!(s, CellSlot::Covered { .. })).count() + } + + #[test] + fn number_formats_apply_to_stored_values() { + // The issue #27 case: a percent renders as its display value, not + // its stored fraction; currency keeps its symbol and grouping; a + // date format keeps the unambiguous ISO rendering. + let wb = Wb { + styles: Some( + r#""#, + ), + ..one_sheet( + r#"0.0751234.546096"#, + ) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["7.5%", "$1,234.50", "2026-03-15"]]); + } + + #[test] + fn unresolvable_numfmt_ids_render_general() { + // Id 5 is not an implied built-in and id 30 is locale-specific: + // with no numFmt element the code is unknown, and guessing (a + // currency format, a date shape) would be worse than General. + let wb = Wb { + styles: Some(r#""#), + ..one_sheet( + r#"1234.51234.5"#, + ) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["1234.5", "1234.5"]]); + } + + #[test] + fn value_types_render_by_their_t_attribute() { + let wb = one_sheet( + r#"1#DIV/0!=sum2026-03-15inline"#, + ); + let doc = parse(&wb.build()).unwrap(); + assert_eq!( + texts(first_table(&doc)), + vec![vec!["TRUE", "#DIV/0!", "=sum", "2026-03-15", "inline"]] + ); + } + + #[test] + fn shared_strings_resolve_including_rich_text_runs() { + let wb = Wb { + shared: Some( + r#"plainrichignored"#, + ), + ..one_sheet( + r#"01"#, + ) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["plain", "rich"]]); + } + + #[test] + fn date1904_serials_shift_epoch() { + let wb = Wb { + styles: Some( + r#""#, + ), + date1904: true, + ..one_sheet(r#"100"#) + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(texts(first_table(&doc)), vec![vec!["1904-04-10"]]); + } + + #[test] + fn merge_extends_past_the_populated_range() { + // Issue #8: the only populated cell anchors F1:O3, so the grid must + // widen to the merge's full 3x10 extent instead of clipping to the + // 1x1 populated range. + let wb = one_sheet( + r#"wide"#, + ); + let doc = parse(&wb.build()).unwrap(); + let table = first_table(&doc); + assert_eq!(table.grid.len(), 3); + assert_eq!(table.grid[1].len(), 10); + let CellSlot::Origin(cell) = &table.grid[0][0] else { + panic!("expected the merge origin at (0,0)"); + }; + assert_eq!((cell.col_span, cell.row_span), (10, 3)); + assert_eq!(covered_count(table), 29); + } + + /// Minimal sheet with a used range at D11:E12 and the given merge. + fn sheet_with_merge(merge_ref: &str) -> Vec { + one_sheet(&format!( + r#"xyzw"# + )) + .build() + } + + #[test] + fn merge_inside_the_used_range_covers_cells() { + let doc = parse(&sheet_with_merge("D11:E11")).unwrap(); + assert_eq!(covered_count(first_table(&doc)), 1); + } + + #[test] + fn merge_outside_the_used_columns_is_ignored() { + // The merge overlaps the used rows but not the used columns; it + // must neither cover cells nor drag the grid out to column A. + let doc = parse(&sheet_with_merge("A1:B12")).unwrap(); + let table = first_table(&doc); + assert_eq!(covered_count(table), 0, "out-of-range merge must not cover cells"); + assert_eq!(table.grid[0].len(), 2); + } + + #[test] + fn hidden_rows_columns_and_sheets_are_omitted() { + // Hidden content is invisible to someone opening the workbook, so + // passing it on would make it look authoritative. One visible sheet + // remains, so no sheet heading is emitted either. + let visible = r#"ahidden colcd"#; + let secret = r#"secret"#; + let wb = Wb { + sheets: vec![("Shown", "", visible), ("Secret", "hidden", secret)], + ..Wb::default() + }; + let doc = parse(&wb.build()).unwrap(); + assert_eq!(doc.blocks.len(), 1, "hidden sheet must add no heading and no table"); + assert_eq!(texts(first_table(&doc)), vec![vec!["a", "c"], vec!["d", ""]]); + } + + #[test] + fn merges_remap_across_hidden_columns() { + // Dropping a hidden column renumbers the grid: a merge spanning it + // comes out one column narrower, not applied at stale indices. + let wb = one_sheet( + r#"mx"#, + ); + let doc = parse(&wb.build()).unwrap(); + let table = first_table(&doc); + let CellSlot::Origin(cell) = &table.grid[0][0] else { + panic!("expected the merge origin at (0,0)"); + }; + assert_eq!((cell.col_span, cell.row_span), (2, 1)); + assert_eq!(table.grid[0].len(), 3); + } + + #[test] + fn merge_origin_in_a_hidden_row_keeps_its_content() { + // The origin row is hidden but the merge survives: its value moves + // to the first surviving position it covers instead of being lost. + let wb = one_sheet( + r#"x"#, + ); + let doc = parse(&wb.build()).unwrap(); + let table = first_table(&doc); + let CellSlot::Origin(cell) = &table.grid[0][0] else { + panic!("expected the merge origin at (0,0)"); + }; + assert_eq!(cell.row_span, 2); + assert_eq!(texts(table)[0][0], "kept"); + } + + #[test] + fn the_grid_budget_spans_the_whole_workbook() { + // Two cells at opposite corners describe the whole sheet, so the + // extent has to be charged before any position is built. + let wb = one_sheet( + r#"ab"#, + ); + let err = parse(&wb.build()).unwrap_err(); + assert!( + matches!(err, ConvertError::ResourceLimit { limit: "max_grid_slots", .. }), + "got {err:?}" + ); + + // Sheets accumulate, so a pile of them cannot each sit under the cap. + let half = r#"ab"#; + let wb = Wb { sheets: vec![("A", "", half), ("B", "", half)], ..Wb::default() }; + let err = parse(&wb.build()).unwrap_err(); + assert!( + matches!(err, ConvertError::ResourceLimit { limit: "max_grid_slots", .. }), + "got {err:?}" + ); + } + + const DATE_ONLY: DateParts = DateParts { date: true, time: false, elapsed: false }; + + #[test] + fn the_fictitious_leap_day_keeps_its_own_value() { + // Serial 60 is 1900-02-29, a day that never existed, and mapping it + // onto a real date would make it indistinguishable from serial 59. + assert_eq!(render_serial(59.0, DATE_ONLY, false), "1900-02-28"); + assert_eq!(render_serial(60.0, DATE_ONLY, false), "60"); + assert_eq!(render_serial(61.0, DATE_ONLY, false), "1900-03-01"); + // A value late on serial 59 still belongs to its own day. + assert_eq!(render_serial(59.9999999, DATE_ONLY, false), "1900-02-28"); + // The 1904 system has no such day. + assert_eq!(render_serial(60.0, DATE_ONLY, true), "1904-03-01"); + } + + #[test] + fn a_sub_day_serial_keeps_the_clock_a_combined_format_names() { + let both = DateParts { date: true, time: true, elapsed: false }; + assert_eq!(render_serial(0.5, both, false), "12:00:00"); + // A date-only format still has nothing to show but the number. + assert_eq!(render_serial(0.5, DATE_ONLY, false), "0.5"); + } +} diff --git a/src/lib.rs b/src/lib.rs index efba6ff8..97099491 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,8 +45,8 @@ pub enum Format { Rtf, /// EPUB 2 and 3 (`.epub`). Epub, - /// Excel workbooks in every container calamine reads: `.xlsx`, `.xlsm`, - /// `.xlsb`, and binary `.xls`. + /// Excel workbooks: `.xlsx`, `.xlsm`, binary `.xlsb`, and legacy + /// OLE-based `.xls`. Excel, /// OpenDocument Spreadsheet (`.ods`). Ods, diff --git a/src/model/table.rs b/src/model/table.rs index 42f16cac..94941580 100644 --- a/src/model/table.rs +++ b/src/model/table.rs @@ -120,6 +120,9 @@ pub struct GridBuilder { /// [`limits::MAX_EXPANSION`] *before* any per-position work so a tiny /// document carrying a huge span cannot force unbounded insertions. expansion: u64, + /// Whether trailing rows holding only covered positions survive + /// [`GridBuilder::finish`]; see [`GridBuilder::keep_covered_tail`]. + keep_covered_tail: bool, } impl GridBuilder { @@ -131,6 +134,14 @@ impl GridBuilder { self.grid.push(Vec::new()); } + /// Treat covered positions as content when trimming trailing rows. A + /// spreadsheet merge region is real extent even where every covered + /// cell is empty; other sources treat such rows as filler and keep the + /// default trim. + pub fn keep_covered_tail(&mut self) { + self.keep_covered_tail = true; + } + fn row_index(&mut self) -> usize { if self.grid.is_empty() { self.grid.push(Vec::new()); @@ -248,7 +259,7 @@ impl GridBuilder { while self.grid.last().is_some_and(|r| { r.iter().all(|s| match s { CellSlot::Origin(c) => c.is_empty(), - CellSlot::Covered { .. } => true, + CellSlot::Covered { .. } => !self.keep_covered_tail, }) }) { self.grid.pop(); diff --git a/src/package/limits.rs b/src/package/limits.rs index f6dba397..1921ebca 100644 --- a/src/package/limits.rs +++ b/src/package/limits.rs @@ -23,6 +23,9 @@ pub const MAX_XML_DEPTH: usize = 256; /// memory test) so a saturating part stays around the archive budget. pub const MAX_XML_NODES: usize = 2_000_000; +/// Maximum grid positions one spreadsheet table may materialize: 4 million. +pub const MAX_GRID_SLOTS: u64 = 4_000_000; + /// Maximum content-bearing cells a repeat expansion may produce per table. pub const MAX_EXPANSION: u64 = 4_000_000; diff --git a/src/package/xml.rs b/src/package/xml.rs index 42af1d8c..e33d1490 100644 --- a/src/package/xml.rs +++ b/src/package/xml.rs @@ -25,6 +25,7 @@ pub mod ns { pub const CHART: &str = "http://schemas.openxmlformats.org/drawingml/2006/chart"; pub const DGM: &str = "http://schemas.openxmlformats.org/drawingml/2006/diagram"; pub const P: &str = "http://schemas.openxmlformats.org/presentationml/2006/main"; + pub const SML: &str = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; pub const PKG_RELS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; pub const OFFICE: &str = "urn:oasis:names:tc:opendocument:xmlns:office:1.0"; diff --git a/tests/fixtures/xlsb/handmade-sheet.xlsb b/tests/fixtures/xlsb/handmade-sheet.xlsb new file mode 100644 index 00000000..e69c2945 Binary files /dev/null and b/tests/fixtures/xlsb/handmade-sheet.xlsb differ diff --git a/tests/gen_fixtures.py b/tests/gen_fixtures.py index 21552bf7..5defd39c 100644 --- a/tests/gen_fixtures.py +++ b/tests/gen_fixtures.py @@ -914,6 +914,112 @@ def merged_xlsx(): ]) +# --------------------------------------------------------------------------- +# Handmade XLSB: binary SpreadsheetML records (MS-XLSB). Exercises the +# variable-length record framing (two-byte ids, multi-byte sizes via a long +# shared string), RK integer/float cells with and without the /100 bit, +# number formats, a hidden row, a hidden column, a hidden sheet, and a merge +# extending past the populated range. + +def xlsb_rec(rec_id, payload=b""): + out = bytearray() + if rec_id < 0x80: + out.append(rec_id) + else: + out.append((rec_id & 0x7F) | 0x80) + out.append(rec_id >> 7) + size = len(payload) + while True: + low = size & 0x7F + size >>= 7 + if size == 0: + out.append(low) + break + out.append(low | 0x80) + return bytes(out) + payload + + +def xlsb_str(s): + data = s.encode("utf-16-le") + return struct.pack("