From 782186878e1a351604826526d142bc2ba73cb303 Mon Sep 17 00:00:00 2001 From: Nicholas023 <498594526@qq.com> Date: Thu, 19 Nov 2026 04:51:15 +0800 Subject: [PATCH 1/3] fix(files): edit rendered source lines without changing literal precedence (#939) --- native/crates/ops-files/src/file_single.rs | 138 +++++++++++++++--- native/crates/ops-files/src/lib.rs | 1 + native/crates/ops-files/src/rendered_text.rs | 144 +++++++++++++++++++ src/projections/native/ranges.ts | 4 + src/projections/native/render.ts | 6 +- test/file-escaped-edit.test.mjs | 89 ++++++++++++ test/native-ranges.test.mjs | 3 +- 7 files changed, 362 insertions(+), 23 deletions(-) create mode 100644 native/crates/ops-files/src/rendered_text.rs create mode 100644 test/file-escaped-edit.test.mjs diff --git a/native/crates/ops-files/src/file_single.rs b/native/crates/ops-files/src/file_single.rs index e7c547eb..3ad06f3d 100644 --- a/native/crates/ops-files/src/file_single.rs +++ b/native/crates/ops-files/src/file_single.rs @@ -77,12 +77,18 @@ pub(super) fn run(request: &Request) -> Result { "edit_file" => { let args: EditArgs = serde_json::from_value(request.args.clone())?; valid_revision(&args.expected_revision, false)?; - write_single( + // Raw bytes stay the matching contract; when oldText does not occur + // in them, exactly one re-interpretation is tried: the reversibly + // escaped rendering the Native reader shows (#939). Raw matches keep + // precedence, so a literal text is never re-read through the decoder. + let rendered_matches = std::cell::Cell::new(0u32); + let mut value = write_single( request, &args.path, &args.expected_revision, args.dry_run, |original| { + rendered_matches.set(0); let text = std::str::from_utf8(original.ok_or_else(|| { Failure::new("NOT_FOUND", "Edit requires an existing file") })?) @@ -98,25 +104,24 @@ pub(super) fn run(request: &Request) -> Result { "oldText must not be empty", )); } - // Count overlapping occurrences as well ("aa" in "aaa" is ambiguous). - let at = text.find(&edit.old_text).ok_or_else(|| { - Failure::new( - "EDIT_CONFLICT", - "oldText was not found; inspect the current file", - ) - })?; - let next_character = text[at..] - .chars() - .next() - .expect("nonempty match") - .len_utf8(); - if text[at + next_character..].contains(&edit.old_text) { - return Err(Failure::new( - "EDIT_CONFLICT", - "oldText is not unique; include more surrounding text", - )); + let decoded = crate::rendered_text::decode_rendered(&edit.old_text); + let (at, length, replacement, rendered) = + match find_unique(text, &edit.old_text) { + Ok(at) => (at, edit.old_text.len(), edit.new_text.clone(), false), + Err(raw) => match find_unique(text, &decoded) { + Ok(at) => ( + at, + decoded.len(), + crate::rendered_text::decode_rendered(&edit.new_text), + true, + ), + Err(_) => return Err(raw), + }, + }; + if rendered { + rendered_matches.set(rendered_matches.get() + 1); } - spans.push((at, at + edit.old_text.len(), edit.new_text.as_str())); + spans.push((at, at + length, replacement)); } spans.sort_by_key(|span| span.0); if spans.windows(2).any(|pair| pair[0].1 > pair[1].0) { @@ -129,13 +134,21 @@ pub(super) fn run(request: &Request) -> Result { let mut cursor = 0; for (start, end, replacement) in spans { next.push_str(&text[cursor..start]); - next.push_str(replacement); + next.push_str(&replacement); cursor = end; } next.push_str(&text[cursor..]); Ok(next.into_bytes()) }, - ) + )?; + let rendered = rendered_matches.get(); + if rendered > 0 + && let Some(items) = value["items"].as_array_mut() + && let Some(item) = items.first_mut() + { + item["renderedMatches"] = json!(rendered); + } + Ok(value) } "copy" | "move" => transfer_file(request), "remove" => remove_file(request), @@ -145,6 +158,27 @@ pub(super) fn run(request: &Request) -> Result { )), } } +/// First occurrence with the existing no-ambiguity rule, shared by the raw and +/// rendered interpretations of oldText. +fn find_unique(text: &str, needle: &str) -> Result { + let at = text.find(needle).ok_or_else(|| { + Failure::new( + "EDIT_CONFLICT", + "oldText was not found; it matches neither the raw file bytes nor the \ + decoded form of the rendered read output; inspect the current file", + ) + })?; + // Count overlapping occurrences as well ("aa" in "aaa" is ambiguous). + let next_character = text[at..].chars().next().expect("nonempty match").len_utf8(); + if text[at + next_character..].contains(needle) { + return Err(Failure::new( + "EDIT_CONFLICT", + "oldText is not unique; include more surrounding text", + )); + } + Ok(at) +} + fn valid_revision(revision: &str, absent: bool) -> Result<()> { if (absent && revision == "absent") || (revision.len() == 64 @@ -831,4 +865,66 @@ mod tests { drop(checked); std::fs::remove_dir_all(root).unwrap(); } + + #[test] + fn old_text_copied_from_the_rendered_reader_edits_after_a_raw_miss() { + let root = std::fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!("dsh-ops-rendered-edit-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let path = root.join("rendered.txt"); + // A raw tab and a raw backslash: the Native reader renders them as \t and \\. + let raw = "alpha\ttab C:\\path\nbeta\n"; + std::fs::write(&path, raw).unwrap(); + let expected_revision = format!("{:x}", Sha256::digest(raw.as_bytes())); + // Exactly the text a model copies out of the rendered read output. + let rendered = r"alpha\ttab C:\\path"; + let value = run(&request( + &root, + "edit_file", + json!({ + "path": path.to_string_lossy(), + "expectedRevision": expected_revision, + "edits": [{"oldText": rendered, "newText": r"alpha\ttab C:\\renamed"}], + }), + )) + .unwrap(); + assert_eq!(value["status"], json!("completed")); + assert_eq!(value["items"][0]["renderedMatches"], json!(1)); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "alpha\ttab C:\\renamed\nbeta\n" + ); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn raw_literals_keep_precedence_over_the_rendered_interpretation() { + let root = std::fs::canonicalize(std::env::temp_dir()) + .unwrap() + .join(format!("dsh-ops-rendered-precedence-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let path = root.join("precedence.txt"); + // The file holds BOTH a literal backslash-t pair and a raw tab; an + // oldText that occurs raw must never be re-read through the decoder. + let raw = "literal\\tab\treal\n"; + std::fs::write(&path, raw).unwrap(); + let value = run(&request( + &root, + "edit_file", + json!({ + "path": path.to_string_lossy(), + "expectedRevision": format!("{:x}", Sha256::digest(raw.as_bytes())), + "edits": [{"oldText": "literal\\tab", "newText": "literal\\kept"}], + }), + )) + .unwrap(); + assert_eq!(value["status"], json!("completed")); + assert_eq!(value["items"][0]["renderedMatches"], json!(Value::Null)); + assert_eq!( + std::fs::read_to_string(&path).unwrap(), + "literal\\kept\treal\n" + ); + std::fs::remove_dir_all(&root).unwrap(); + } } diff --git a/native/crates/ops-files/src/lib.rs b/native/crates/ops-files/src/lib.rs index d03bc170..9ba5de64 100644 --- a/native/crates/ops-files/src/lib.rs +++ b/native/crates/ops-files/src/lib.rs @@ -4,6 +4,7 @@ pub mod cache_io; mod file; pub mod file_path; pub mod paths; +mod rendered_text; #[cfg(all(test, unix))] use cap_std::ambient_authority; #[cfg(unix)] diff --git a/native/crates/ops-files/src/rendered_text.rs b/native/crates/ops-files/src/rendered_text.rs new file mode 100644 index 00000000..53d89437 --- /dev/null +++ b/native/crates/ops-files/src/rendered_text.rs @@ -0,0 +1,144 @@ +//! Decode the reversible rendering the Native reader shows for source lines. +//! +//! The encoder is "src/projections/native/line-encoding.ts"; this module is +//! its exact inverse, never a second encoding policy. ops_edit_file consults +//! it only after a raw match has failed (#939), so literal text keeps +//! precedence. + +/// Interpret "text" as one rendered source line: a doubled backslash and the +/// JSON-style escapes (\b, \t, \n, \f, \r, \uXXXX) become their raw +/// characters. Unknown escapes and plain text stay byte-identical, so a string +/// that never came from the renderer decodes to itself and a half escape +/// invents nothing. +pub fn decode_rendered(text: &str) -> String { + let bytes = text.as_bytes(); + let mut out = String::with_capacity(bytes.len()); + let mut at = 0; + while at < bytes.len() { + if bytes[at] == b'\\' && at + 1 < bytes.len() { + match bytes[at + 1] { + b'\\' => { + out.push('\\'); + at += 2; + continue; + } + b'b' => { + out.push('\u{0008}'); + at += 2; + continue; + } + b't' => { + out.push('\t'); + at += 2; + continue; + } + b'n' => { + out.push('\n'); + at += 2; + continue; + } + b'f' => { + out.push('\u{000c}'); + at += 2; + continue; + } + b'r' => { + out.push('\r'); + at += 2; + continue; + } + b'u' if at + 6 <= bytes.len() + && bytes[at + 2..at + 6].iter().all(u8::is_ascii_hexdigit) => + { + let code = u32::from_str_radix(&text[at + 2..at + 6], 16).unwrap_or(0); + match char::from_u32(code) { + Some(decoded) => out.push(decoded), + // Surrogate halves never appear in rendered text; keep + // the six characters literal instead of inventing one. + None => out.push_str(&text[at..at + 6]), + } + at += 6; + continue; + } + _ => {} + } + } + // Copy one whole UTF-8 character: astral text stays intact. + let width = character_width(bytes[at]); + out.push_str(&text[at..at + width]); + at += width; + } + out +} + +fn character_width(lead: u8) -> usize { + match lead { + 0x00..=0x7f => 1, + 0xc0..=0xdf => 2, + 0xe0..=0xef => 3, + 0xf0..=0xf7 => 4, + // A valid &str never offers any other lead byte. + _ => 1, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rendered_escapes_decode_to_their_exact_source_characters() { + assert_eq!(decode_rendered(r"a\tb"), "a\tb"); + assert_eq!(decode_rendered(r"C:\\new"), "C:\\new"); + assert_eq!(decode_rendered(r"\b\f\r\n"), "\u{0008}\u{000c}\r\n"); + assert_eq!(decode_rendered(r"\u0000"), "\0"); + assert_eq!(decode_rendered(r"\u2028text"), "\u{2028}text"); + assert_eq!(decode_rendered(r"\u0085"), "\u{0085}"); + } + + #[test] + fn plain_text_unknown_and_broken_escapes_stay_byte_identical() { + assert_eq!(decode_rendered("plain 中文 😀"), "plain 中文 😀"); + assert_eq!(decode_rendered(r"\q\z"), "\\q\\z"); + assert_eq!(decode_rendered(r"\uzzzz"), "\\uzzzz"); + assert_eq!(decode_rendered(r"\ud800"), "\\ud800"); + assert_eq!(decode_rendered(r"\u12"), "\\u12"); + assert_eq!(decode_rendered("trailing\\"), "trailing\\"); + assert_eq!(decode_rendered("back\\中"), "back\\中"); + } + + #[test] + fn decoding_the_renderer_output_restores_every_source_sample() { + // The same escape classes the TS encoder renders (see its suite); a + // decoded rendering must be the raw source again. + for source in [ + "alpha\ttab C:\\path\nbeta\n", + "literal\\tab\treal\n", + "\u{7f}\u{85}\u{2028}\u{2029}\0", + ] { + let rendered = encode_for_test(source); + assert_ne!(rendered, source, "the sample must actually be rendered"); + assert_eq!(decode_rendered(&rendered), source); + } + } + + /// A mirror of the projection encoder, for round-trip coverage only. + fn encode_for_test(source: &str) -> String { + let mut out = String::new(); + for ch in source.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '\u{0008}' => out.push_str("\\b"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\u{000c}' => out.push_str("\\f"), + '\r' => out.push_str("\\r"), + c if (c as u32) < 0x20 || matches!(c as u32, 0x7f | 0x85 | 0x2028 | 0x2029) => { + out.push_str(&format!("\\u{:04x}", c as u32)); + } + c => out.push(c), + } + } + out + } +} diff --git a/src/projections/native/ranges.ts b/src/projections/native/ranges.ts index 3342fee6..fcf8793c 100644 --- a/src/projections/native/ranges.ts +++ b/src/projections/native/ranges.ts @@ -46,6 +46,10 @@ export function compactRanges(input: Row, value: Row): string[] | undefined { if (!item.lines.length) output.push(item.total === 0 ? '(empty file)' : '(no lines in this range)') output.push(`[revision=${item.revision}; ${item.complete ? 'end of file; complete=true' : `more lines; nextOffset=${item.next}`}]`) } + // The compact reader states the escape contract once for the whole result (#939). + const escaped = observed.reduce( + (count, item) => count + item.lines.filter(line => encodeSourceLine(line.text) !== line.text).length, 0) + if (escaped) output.push(`[Notice: ${escaped} rendered line(s) use reversible escapes; ops_edit_file matches oldText as raw bytes first, then as this displayed form decoded.]`) output.push(`[Notice: ${observed.length}/${input.files.length} requested ranges returned.]`) return output } diff --git a/src/projections/native/render.ts b/src/projections/native/render.ts index 3ba68ba5..99030dc0 100644 --- a/src/projections/native/render.ts +++ b/src/projections/native/render.ts @@ -89,6 +89,10 @@ function inspect(args: Row, value: Row): string[] { } if (!group.length) output.push(value.lineCount === 0 ? '(empty file)' : '(no lines in this range)') } + // Escaped display is a contract the model must be able to round-trip: say so + // once per result, exactly when it matters (#939). + const escaped = items.filter(item => typeof item.text === 'string' && contentLine(item.text) !== item.text) + if (escaped.length) output.push(`[Notice: ${escaped.length} rendered line(s) use reversible escapes; ops_edit_file matches oldText as raw bytes first, then as this displayed form decoded.]`) output.push(...notices(value, items.length, textTotal !== undefined && value.total === textTotal ? ['total'] : [])) return output } @@ -112,7 +116,7 @@ const mutations = new Set(FILE_MUTATIONS) function receiptItems(value: Row): string[] { return rows(value.items).map(item => `${label(item.path ?? '(path unavailable)')}: ${facts(item, [ 'status', 'removed', 'changed', 'revision', 'revisionBefore', 'revisionAfter', 'bytes', 'bytesBefore', 'bytesAfter', - 'lineCountBefore', 'lineCount', 'replacements', 'metadata', 'metadataWarnings', 'error', + 'lineCountBefore', 'lineCount', 'replacements', 'renderedMatches', 'metadata', 'metadataWarnings', 'error', ])}`) } function receipts(tool: string, value: Row): string[] { diff --git a/test/file-escaped-edit.test.mjs b/test/file-escaped-edit.test.mjs new file mode 100644 index 00000000..83295ffe --- /dev/null +++ b/test/file-escaped-edit.test.mjs @@ -0,0 +1,89 @@ +/** #939: the Native reader renders backslashes and controls with reversible JSON + * escapes. ops_edit_file matches oldText against the raw bytes first; when that + * misses, exactly one re-interpretation is tried - the rendered form decoded - and + * the receipt says so. Raw matches keep precedence, so literal text is never + * re-read through the decoder. */ +import fs from 'node:fs' +import path from 'node:path' +import { createHash } from 'node:crypto' +import { encodeSourceLine } from '../lib/file/native/line-encoding.js' +import { renderNative } from '../lib/file/native/render.js' +import { assert, test, report, withTempDir } from './lib/harness.mjs' +import { callTool, envelopeOf, requireWorker } from './lib/boundary.mjs' +requireWorker('file-escaped-edit') +const call = async (tool, args, root) => envelopeOf(await callTool(tool, args, { root }), assert) +const rev = file => createHash('sha256').update(fs.readFileSync(file)).digest('hex') + +await test('oldText copied from the rendered line edits after the raw miss', async () => { + await withTempDir('file-escaped-edit', async root => { + const target = path.join(root, 'rendered.txt') + fs.writeFileSync(target, 'alpha\ttab C:\\path\nbeta\n') + const revision = rev(target) + const rendered = encodeSourceLine('alpha\ttab C:\\path') + assert.notEqual(rendered, 'alpha\ttab C:\\path', 'the raw line must be rendered escaped before the model sees it') + const body = await call('edit_file', { path: target, expectedRevision: revision, + edits: [{ oldText: rendered, newText: encodeSourceLine('alpha\ttab C:\\renamed') }] }, root) + assert.equal(body.status, 'completed', JSON.stringify(body)) + assert.equal(body.items[0].renderedMatches, 1, JSON.stringify(body)) + assert.equal(fs.readFileSync(target, 'utf8'), 'alpha\ttab C:\\renamed\nbeta\n') + }) +}) + +await test('raw literal matches keep precedence and never decode newText', async () => { + await withTempDir('file-escaped-precedence', async root => { + const target = path.join(root, 'precedence.txt') + // The file holds BOTH a literal backslash-t pair and a raw tab. + fs.writeFileSync(target, 'literal\\tab\treal\n') + const body = await call('edit_file', { path: target, expectedRevision: rev(target), + edits: [{ oldText: 'literal\\tab', newText: 'literal\\kept' }] }, root) + assert.equal(body.status, 'completed', JSON.stringify(body)) + assert.equal(body.items[0].renderedMatches, undefined, 'a raw match must not count as rendered') + assert.equal(fs.readFileSync(target, 'utf8'), 'literal\\kept\treal\n') + }) +}) + +await test('ambiguous or absent decodes stay honest EDIT_CONFLICTs', async () => { + await withTempDir('file-escaped-conflict', async root => { + const target = path.join(root, 'conflict.txt') + fs.writeFileSync(target, 'a\tb a\tb\n') + const duplicated = await call('edit_file', { path: target, expectedRevision: rev(target), + edits: [{ oldText: encodeSourceLine('a\tb'), newText: 'x' }] }, root) + assert.equal(duplicated.error?.code, 'EDIT_CONFLICT', JSON.stringify(duplicated)) + assert.equal(duplicated.error.effectState, 'none') + fs.writeFileSync(target, 'plain line\n') + const missing = await call('edit_file', { path: target, expectedRevision: rev(target), + edits: [{ oldText: encodeSourceLine('absent\\q'), newText: 'x' }] }, root) + assert.equal(missing.error?.code, 'EDIT_CONFLICT', JSON.stringify(missing)) + assert.match(missing.error.message, /rendered|escaped/i, 'the refusal must name the second interpretation') + assert.equal(fs.readFileSync(target, 'utf8'), 'plain line\n') + }) +}) + +await test('write_file stays byte-verbatim and never decodes content', async () => { + await withTempDir('file-escaped-write', async root => { + const target = path.join(root, 'verbatim.txt') + const body = await call('write_file', { path: target, expectedRevision: 'absent', content: 'keep \\t literal\n' }, root) + assert.equal(body.status, 'completed', JSON.stringify(body)) + assert.equal(fs.readFileSync(target, 'utf8'), 'keep \\t literal\n') + }) +}) + +await test('the rendered receipt and read output state the escape contract', async () => { + const file = path.resolve('rendered-source.txt') + const value = { status: 'completed', path: file, revision: 'a'.repeat(64), lineCount: 1, nextOffset: 2, + complete: true, truncated: false, failed: 0, + items: [{ path: file, line: 1, text: 'alpha\ttab' }] } + const mutated = { status: 'completed', phase: 'verify', complete: true, changed: 1, dryRun: false, + items: [{ path: file, status: 'completed', changed: 1, renderedMatches: 1 }] } + const inspection = renderNative('inspect_local_file', { path: file, limit: 1 }, value) + assert.match(inspection, /reversible escapes/, 'reads must warn when escapes are displayed') + assert.doesNotMatch(renderNative('inspect_local_file', { path: file, limit: 1 }, + { ...value, items: [{ path: file, line: 1, text: 'plain' }] }), /reversible escapes/) + const batch = { status: 'completed', total: 1, nextIndex: 1, pending: 0, failed: 0, truncated: false, items: [value] } + assert.match(renderNative('read_files', { files: [{ path: file, limit: 1 }] }, batch), /reversible escapes/) + assert.match(renderNative('edit_file', { path: file }, mutated), /renderedMatches=1/, 'the receipt must surface rendered matches') + assert.doesNotMatch(renderNative('edit_file', { path: file }, + { ...mutated, items: [{ path: file, status: 'completed', changed: 1 }] }), /renderedMatches/) +}) + +report('file-escaped-edit') diff --git a/test/native-ranges.test.mjs b/test/native-ranges.test.mjs index 33e8b41d..baaa76aa 100644 --- a/test/native-ranges.test.mjs +++ b/test/native-ranges.test.mjs @@ -24,7 +24,8 @@ await test('real bounded ranges keep exact body, shared path base, full revision assert.match(text, /more lines; nextOffset=3/); assert.match(text, /\(empty file\)/) assert.match(text, /end of file; complete=true/); assert.match(text, /3\/3 requested ranges returned/) assert.doesNotMatch(text, /worker-limit|bytes=|lineCount=|matched=/) - assert.equal(text.split('\n').filter(line => line.startsWith('[Notice:')).length, 1, 'one aggregate notice covers all fully returned ranges') + assert.equal(text.split('\n').filter(line => line.startsWith('[Notice:')).length, 2, 'one escape notice plus one aggregate notice cover all fully returned ranges') + assert.match(text, /rendered line\(s\) use reversible escapes/, 'escaped lines state the round-trip contract once') assert.equal(JSON.stringify(value), before) })) await test('different parents retain absolute quoted paths; path control characters cannot inject headers', () => { From db31e285df07fb6a9e3b909974778a65df8c7849 Mon Sep 17 00:00:00 2001 From: Nicholas023 <498594526@qq.com> Date: Thu, 19 Nov 2026 04:51:15 +0800 Subject: [PATCH 2/3] fix(documents): resolve indirect PDF stream lengths (#788) --- .../crates/ops-documents/src/pdf_container.rs | 74 ++++++++++++++--- .../ops-documents/src/pdf_container_tests.rs | 81 +++++++++++++++++++ test/pdf-indirect-length.test.mjs | 78 ++++++++++++++++++ 3 files changed, 222 insertions(+), 11 deletions(-) create mode 100644 test/pdf-indirect-length.test.mjs diff --git a/native/crates/ops-documents/src/pdf_container.rs b/native/crates/ops-documents/src/pdf_container.rs index d5a2debb..bef2d3ba 100644 --- a/native/crates/ops-documents/src/pdf_container.rs +++ b/native/crates/ops-documents/src/pdf_container.rs @@ -1,4 +1,4 @@ -//! A deliberately restricted PDF container: one classic xref table, direct stream lengths, no recovery. +//! A deliberately restricted PDF container: one classic xref table, lengths resolved through that xref, no recovery. use crate::pdf_syntax::{Dict, Lexer, Value, bad, limit, unsupported, white}; use ops_protocol::Result; use std::collections::{BTreeMap, BTreeSet}; @@ -6,6 +6,8 @@ pub const MAX_FILE: usize = 16 * 1024 * 1024; pub const MAX_OBJECTS: usize = 10_000; pub const MAX_STREAM: usize = 1024 * 1024; pub type ObjectId = (u32, u16); +/// The classic xref table: object number to (byte offset, generation, active). +type XrefEntries = BTreeMap; #[derive(Debug)] pub struct Object<'a> { pub value: Value, @@ -82,11 +84,62 @@ fn footer(data: &[u8]) -> Result<(usize, usize)> { } Ok((xref, footer_start)) } +/// Resolve one stream Length (#788): a direct bounded integer, or one indirect +/// reference whose xref entry names a plain integer object. The probe re-lexes +/// the provider's exact header, so absent, free, chained, non-integer and +/// identity-mismatched targets fail closed; the caller's node budget pays for +/// the probe, keeping the whole parse bounded. No recovery, no endstream scan. +fn stream_length( + data: &[u8], + entries: &XrefEntries, + dict: &Dict, + nodes: &mut usize, +) -> Result { + let length = match get(dict, b"Length")? { + Value::Ref(number, generation) => { + let (offset, entry_generation, active) = + *entries.get(&(*number as usize)).ok_or_else(|| { + bad("PDF stream Length does not name an xref entry") + })?; + if !active || entry_generation != *generation { + return Err(bad( + "PDF stream Length must name an active xref entry with a matching generation", + )); + } + if !data.get(offset).is_some_and(u8::is_ascii_digit) { + return Err(bad("PDF stream Length xref must point exactly to an object header")); + } + let mut lexer = Lexer { + data, + at: offset, + nodes: *nodes, + }; + if lexer.unsigned()? != *number as usize || lexer.unsigned()? != *generation as usize { + return Err(bad("PDF stream Length object identity mismatch")); + } + lexer.keyword(b"obj")?; + let resolved = lexer + .value(0)? + .integer() + .map_err(|_| bad("PDF stream Length must resolve to a bounded integer object"))?; + lexer.keyword(b"endobj")?; + *nodes = lexer.nodes; + resolved + } + value => value.integer()?, + }; + if length > MAX_STREAM { + return Err(limit("PDF encoded stream exceeds 1 MiB")); + } + Ok(length) +} + fn object<'a>( data: &'a [u8], offset: usize, id: ObjectId, nodes: usize, + entries: &XrefEntries, ) -> Result<(Object<'a>, usize, usize)> { if !data.get(offset).is_some_and(u8::is_ascii_digit) { return Err(bad("PDF xref must point exactly to an object header")); @@ -104,11 +157,10 @@ fn object<'a>( let mut stream = None; let token = lexer.word()?; if token == b"stream" { - let length = get(value.dict()?, b"Length")?.integer()?; - if length > MAX_STREAM { - return Err(limit("PDF encoded stream exceeds 1 MiB")); - } - // Indirect Length is declined before reading stream bytes; no searching for endstream. + let mut nodes = lexer.nodes; + let length = stream_length(lexer.data, entries, value.dict()?, &mut nodes)?; + lexer.nodes = nodes; + // The Length is fully resolved before stream bytes are read; no searching for endstream. if lexer.data.get(lexer.at..lexer.at + 2) == Some(b"\r\n") { lexer.at += 2; } else if lexer.data.get(lexer.at) == Some(&b'\n') { @@ -228,16 +280,16 @@ pub fn parse(data: &[u8]) -> Result> { return Err(bad("Unexpected PDF data between trailer and final footer")); } let mut ordered = Vec::new(); - for (number, (offset, generation, active)) in entries { - if !active { + for (number, (offset, generation, active)) in &entries { + if !*active { continue; } - if offset < 9 || offset >= xref { + if *offset < 9 || *offset >= xref { return Err(bad( "Active PDF object offset is outside the object section", )); } - ordered.push((offset, (number as u32, generation))); + ordered.push((*offset, (*number as u32, *generation))); } ordered.sort_unstable(); let mut previous = 0; @@ -258,7 +310,7 @@ pub fn parse(data: &[u8]) -> Result> { "Unindexed data or overlapping comments in PDF object section", )); } - let (item, end, count) = object(&data[..xref], offset, id, nodes)?; + let (item, end, count) = object(&data[..xref], offset, id, nodes, &entries)?; previous = end; nodes = count; objects.insert(id, item); diff --git a/native/crates/ops-documents/src/pdf_container_tests.rs b/native/crates/ops-documents/src/pdf_container_tests.rs index 81d5f979..bba6bc86 100644 --- a/native/crates/ops-documents/src/pdf_container_tests.rs +++ b/native/crates/ops-documents/src/pdf_container_tests.rs @@ -337,3 +337,84 @@ fn stream_roots_reserved_generations_and_direct_oversize_decode_are_refused() { }; assert_eq!(decode(&raw).unwrap_err().code, "OUTPUT_LIMIT"); } + +#[test] +fn indirect_stream_lengths_resolve_through_the_xref_without_recovery() { + // #788: Word and LibreOffice emit "/Length N G R". The integer provider may + // sit at any xref offset; one bounded probe re-lexes its exact header. + let pdf = fixture(&[ + b"<<>>", + b"9", + b"<< /Length 2 0 R >>\nstream\n(Hi) Tj\n\nendstream", + ]); + let document = parse(&pdf).unwrap(); + assert_eq!( + document.object((3, 0)).unwrap().stream, + Some(b"(Hi) Tj\n\n".as_slice()) + ); + // The provider may also follow the stream object in file order. + let reversed = fixture(&[ + b"<<>>", + b"<< /Length 3 0 R >>\nstream\nxy\nendstream", + b"2", + ]); + assert_eq!( + parse(&reversed).unwrap().object((2, 0)).unwrap().stream, + Some(b"xy".as_slice()) + ); + // Only a plain integer object resolves: absent, dictionary, chained, + // generation-mismatched and oversized providers all fail closed. + rejected( + &fixture(&[b"<<>>", b"9", b"<< /Length 4 0 R >>\nstream\nxy\nendstream"]), + "DOCUMENT_PARSE_ERROR", + ); + rejected( + &fixture(&[b"<<>>", b"<<>>", b"<< /Length 2 0 R >>\nstream\nxy\nendstream"]), + "DOCUMENT_PARSE_ERROR", + ); + rejected( + &fixture(&[ + b"<<>>", + b"3 0 R", + b"<< /Length 2 0 R >>\nstream\nxy\nendstream", + b"9", + ]), + "DOCUMENT_PARSE_ERROR", + ); + rejected( + &fixture(&[b"<<>>", b"9", b"<< /Length 2 1 R >>\nstream\nxy\nendstream"]), + "DOCUMENT_PARSE_ERROR", + ); + rejected( + &fixture(&[ + b"<<>>", + b"1048577", + b"<< /Length 2 0 R >>\nstream\nxy\nendstream", + ]), + "OUTPUT_LIMIT", + ); + // A free xref entry is never a Length provider. + let mut free = fixture(&[b"<<>>", b"9", b"<< /Length 2 0 R >>\nstream\nxy\nendstream"]); + let at = free.windows(7).position(|w| w == b"2 0 obj").unwrap(); + free = replace( + &free, + format!("{at:010} 00000 n ").as_bytes(), + format!("{at:010} 65535 f ").as_bytes(), + ); + rejected(&free, "DOCUMENT_PARSE_ERROR"); + // The probe runs before the ordered scan reaches a duplicated offset and + // refuses a header that disagrees with the referenced identity. + let mut aliased = fixture(&[ + b"<<>>", + b"<< /Length 3 0 R >>\nstream\nxy\nendstream", + b"2", + ]); + let stream_at = aliased.windows(7).position(|w| w == b"2 0 obj").unwrap(); + let provider_at = aliased.windows(7).position(|w| w == b"3 0 obj").unwrap(); + aliased = replace( + &aliased, + format!("{provider_at:010} 00000 n ").as_bytes(), + format!("{stream_at:010} 00000 n ").as_bytes(), + ); + rejected(&aliased, "DOCUMENT_PARSE_ERROR"); +} diff --git a/test/pdf-indirect-length.test.mjs b/test/pdf-indirect-length.test.mjs new file mode 100644 index 00000000..f17070fa --- /dev/null +++ b/test/pdf-indirect-length.test.mjs @@ -0,0 +1,78 @@ +/** #788: an indirect /Length "N G R" resolves through the same classic xref table. + * No endstream scanning, no recovery: absent, free, chained, non-integer or oversized + * providers keep failing closed exactly like an invalid direct length. */ +import fs from 'node:fs' +import path from 'node:path' +import { assert, test, report, withTempDir } from './lib/harness.mjs' +import { callWorker, request, envelopeOf, requireWorker } from './lib/boundary.mjs' +import { pdf, show } from './lib/pdf-fixture.mjs' +requireWorker('pdf-indirect-length') + +const catalog = '<< /Type /Catalog /Pages 2 0 R >>' +const font = '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>' +const pages = kids => '<< /Type /Pages /Kids [' + kids.join(' ') + '] /Count ' + kids.length + ' >>' +const page = ref => '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ' + ref + ' >>' +const direct = content => '<< /Length ' + Buffer.byteLength(content) + ' >>\nstream\n' + content + '\nendstream' +const indirect = (content, ref) => '<< /Length ' + ref + ' >>\nstream\n' + content + '\nendstream' + +async function run(root, args) { + const payload = { version: 1, requestId: 'pdf-indirect-test', component: 'file', + budget: { timeoutMs: 30000, maxBytes: 131072 }, + request: request('read_pdf', args, { root }) } + return envelopeOf(await callWorker(JSON.stringify(payload)), assert) +} +const failure = (body, code) => { + assert.equal(body.error?.code, code, JSON.stringify(body)) + assert.equal(body.error.effectState, 'none') + assert.notEqual(body.complete, true) +} +const fileAt = (root, value) => { const file = path.join(root, 'document.pdf'); fs.writeFileSync(file, value); return file } + +await test('indirect stream lengths read like direct ones in both file orders', async () => { + await withTempDir('pdf-indirect', async root => { + // Objects: 4 is the page, 5 its content stream, 6 the length provider that follows it. + const providerAfter = pdf([catalog, pages(['4 0 R']), font, page('5 0 R'), + indirect(show('later provider'), '6 0 R'), String(Buffer.byteLength(show('later provider')))]) + const after = await run(root, { path: fileAt(root, providerAfter) }) + assert.equal(after.error, undefined, JSON.stringify(after)) + assert.equal(after.status, 'completed'); assert.equal(after.complete, true) + assert.deepEqual(after.items.map(i => i.text), ['later provider']) + // Provider object 5 sits before the stream object 6 in file order. + const providerBefore = pdf([catalog, pages(['4 0 R']), font, page('6 0 R'), + String(Buffer.byteLength(show('earlier provider'))), indirect(show('earlier provider'), '5 0 R')]) + const before = await run(root, { path: fileAt(root, providerBefore) }) + assert.equal(before.error, undefined, JSON.stringify(before)) + assert.deepEqual(before.items.map(i => i.text), ['earlier provider']) + }) +}) + +await test('direct and indirect lengths coexist in one document', async () => { + await withTempDir('pdf-indirect-mixed', async root => { + const one = show('direct page'), two = show('indirect page') + const mixed = pdf([catalog, pages(['4 0 R', '6 0 R']), font, page('5 0 R'), direct(one), + page('7 0 R'), indirect(two, '8 0 R'), String(Buffer.byteLength(two))]) + const body = await run(root, { path: fileAt(root, mixed) }) + assert.equal(body.error, undefined, JSON.stringify(body)) + assert.deepEqual(body.items.map(i => i.text), ['direct page', 'indirect page']) + }) +}) + +await test('absent, non-integer, mismatched, chained and oversized providers fail closed', async () => { + await withTempDir('pdf-indirect-negative', async root => { + const text = show('safe') + const cases = [ + ['9 0 R', String(Buffer.byteLength(text)), 'DOCUMENT_PARSE_ERROR'], + ['3 0 R', font, 'DOCUMENT_PARSE_ERROR'], + ['6 1 R', String(Buffer.byteLength(text)), 'DOCUMENT_PARSE_ERROR'], + ['2 0 R', '2 0 R', 'DOCUMENT_PARSE_ERROR'], + ] + for (const [ref, provider, code] of cases) { + const bytes = pdf([catalog, pages(['4 0 R']), font, page('5 0 R'), indirect(text, ref), provider]) + failure(await run(root, { path: fileAt(root, bytes) }), code) + } + failure(await run(root, { path: fileAt(root, pdf([catalog, pages(['4 0 R']), font, page('5 0 R'), + indirect(show('x'), '6 0 R'), String(1024 * 1024 + 1)])) }), 'OUTPUT_LIMIT') + }) +}) + +report('pdf-indirect-length') From a6398b0e18deb92627f70956259084e8b5b174c9 Mon Sep 17 00:00:00 2001 From: Nicholas023 <498594526@qq.com> Date: Thu, 19 Nov 2026 10:15:34 +0800 Subject: [PATCH 3/3] docs: record file edit and PDF contract fixes (#939, #788) --- docs/CHANGELOG.md | 5 +++++ docs/file-batching.md | 6 ++++-- docs/native-kernel.md | 2 +- docs/native-ranges.md | 2 +- "docs/\345\216\237\347\224\237\345\206\205\346\240\270.md" | 2 +- ...\266\346\211\271\351\207\217\346\223\215\344\275\234.md" | 6 ++++-- ...\221\346\226\207\344\273\266\350\214\203\345\233\264.md" | 2 +- 7 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d66194e1..2fb24f37 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,11 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Replace the `global.forceDefaultOpsPtc` switch with one explicit action: the row's detail page carries a **Make OPS (PTC) the default** button that writes `selectedDefault: ops-ptc` once and answers `applied`/`no-update`/`unavailable`/`failed`. Apply, remount and unload no longer write the default, so a preset chosen on the settings page is not overridden on the next apply, and a saved `global.forceDefaultOpsPtc` key still loads with no effect. - 用一次显式操作取代 `global.forceDefaultOpsPtc` 开关:行详情页有一个「设为 OPS (PTC) 默认」按钮,点一次写 `selectedDefault: ops-ptc`,并回报 `applied`/`no-update`/`unavailable`/`failed`。apply、重挂载与卸载都不再改写默认,设置页选定的预设不会在下一次 apply 被覆盖;已保存的 `global.forceDefaultOpsPtc` 键仍可加载但不再生效。 +- `ops_edit_file` now accepts text copied from the reversibly escaped Native reader: raw `oldText` matches keep precedence, and only a raw miss tries the decoded rendering. The fallback also decodes `newText`, reports `renderedMatches`, and preserves ambiguity/CAS failures; raw edits and `write_file` remain byte-verbatim (#939). +- `ops_edit_file` 现在可接受从可逆转义的 Native 读取输出中复制的文本:优先匹配原始 `oldText`,只有原始匹配失败才尝试解码后的渲染形式。回退同样解码 `newText` 并返回 `renderedMatches`,歧义与 CAS 拒绝保持不变;原始编辑与 `write_file` 仍逐字节处理(#939)。 +- The bounded classic-xref PDF reader resolves indirect `/Length` values in either object order while keeping cycle, depth, type, size and mismatch refusals. Direct lengths and the existing stream/xref budgets are unchanged (#788). +- 有界的经典 xref PDF 读取器现在可按任意对象顺序解析间接 `/Length`,同时保留循环、深度、类型、大小与不一致拒绝。直接长度及现有流/xref 预算不变(#788)。 + ## [0.5.5] — 2026-11-13 — Patch release - A loader-less PTC scope no longer registers the configured `allowedTools` pool: with dynamic loading off nothing could ever select those APIs, and a host assembly without a step tool set declared all sixteen in `tools:sdk`. Resident publication, the default inspect + loader set and selection behavior are unchanged (#887). diff --git a/docs/file-batching.md b/docs/file-batching.md index e0b77636..9c065eb0 100644 --- a/docs/file-batching.md +++ b/docs/file-batching.md @@ -58,8 +58,10 @@ structured. `ops_edit_file` applies 1-100 edits to **one** UTF-8 file, atomically. Supply `path`, the exact SHA-256 `expectedRevision` from a read/stat, and `edits: [{oldText,newText}]`. Each nonempty -`oldText` must occur exactly once in the same original bytes; overlaps, ambiguity or stale -revision reject all edits before publication. Empty `newText` deletes the match. Preserve +`oldText` must occur exactly once in the same original bytes. If that raw match fails, one +fallback decodes the reversible Native rendering of both `oldText` and `newText`; a raw match +never decodes either field, and a fallback receipt counts `renderedMatches`. Overlaps, ambiguity +or stale revision reject all edits before publication. Empty `newText` deletes the match. Preserve CRLF/BOM explicitly. `dryRun: true` previews the same receipt without writing; read-only policy still rejects mutation tools. This is not a cross-file transaction. diff --git a/docs/native-kernel.md b/docs/native-kernel.md index 7916dc27..ec443cda 100644 --- a/docs/native-kernel.md +++ b/docs/native-kernel.md @@ -275,7 +275,7 @@ reference pages retains the declared main-body scope independently of stored-row The internal PDF container parser accepts a restricted PDF 1.0-1.7 subset: one classic xref table, an exact final startxref/EOF footer, unique dictionary keys and object -identities, and directly declared stream lengths. Every active xref offset must point +identities, and stream lengths declared either directly or through an indirect reference resolved by the classic xref table. Every active xref offset must point exactly at its object header. Objects cannot overlap; gaps must contain only whitespace or comments. This deliberately rejects incremental updates, encryption, object/xref streams, recovery scans and ambiguous duplicate data. `/Encrypt`, `/ObjStm` and `/XRef` diff --git a/docs/native-ranges.md b/docs/native-ranges.md index 1bc2fb1b..c5b524a7 100644 --- a/docs/native-ranges.md +++ b/docs/native-ranges.md @@ -12,7 +12,7 @@ A fully returned `ops_read_files` range uses one path header, exact numbered tex [revision=cfd1bc81432fd6b88748b827f46bb4242db17cf8655cf4ea90f3ba0e62065355; more lines; nextOffset=3] ``` -The characters after `N|` are the source line, including leading spaces. Line bodies use reversible JSON escapes: backslashes are doubled, controls and Unicode line separators are escaped, and quotes remain literal. For example, an actual tab appears as `\t`, while the two source characters backslash-plus-t appear as `\\t`. This shared encoding applies to compact ranges and ordinary numbered source output, and prevents source text from forging extra rows or headers. LF/CRLF accounting remains the worker's physical-line accounting. +The characters after `N|` are the source line, including leading spaces. Line bodies use reversible JSON escapes: backslashes are doubled, controls and Unicode line separators are escaped, and quotes remain literal. For example, an actual tab appears as `\t`, while the two source characters backslash-plus-t appear as `\\t`. This shared encoding applies to compact ranges and ordinary numbered source output, and prevents source text from forging extra rows or headers. LF/CRLF accounting remains the worker's physical-line accounting. `ops_edit_file` first matches literal bytes; only after a raw miss may it decode this rendered form for `oldText` and `newText`, with `renderedMatches` in the receipt. It never decodes a raw match or `write_file` content. “More lines” means the file has not been exhausted; use the stated `nextOffset`. “End of file; complete=true” means this range reached the end. The aggregate “requested ranges returned” receipt means every requested range body was returned, not that every whole file was read. Full revisions, total line counts and per-file continuation remain visible. Byte sizes and duplicate counters remain available in canonical/PTC data. Per-file revision receipts omit the repeated `Notice: ` prefix and terminal period; the aggregate `Notice` and every revision/continuation/completeness fact remain intact. diff --git "a/docs/\345\216\237\347\224\237\345\206\205\346\240\270.md" "b/docs/\345\216\237\347\224\237\345\206\205\346\240\270.md" index 81a1655d..e714870a 100644 --- "a/docs/\345\216\237\347\224\237\345\206\205\346\240\270.md" +++ "b/docs/\345\216\237\347\224\237\345\206\205\346\240\270.md" @@ -216,7 +216,7 @@ FILE_CHANGED 始终使所请求页不完整。引用页生成的 source 类型 ## PDF 解析基础(15c/16) 内部 PDF 容器解析器仅接受受限的 PDF 1.0–1.7 子集:单个传统 xref 表、 -准确的末尾 startxref/EOF、唯一字典键及对象标识,以及直接整数形式的流长度。 +准确的末尾 startxref/EOF、唯一字典键及对象标识,以及直接整数或经经典 xref 表解析的间接引用形式的流长度。 每个有效 xref 偏移必须准确指向对象头;对象不可重叠,间隙只能包含空白或注释。 增量更新、加密、对象流/xref 流、恢复扫描及重复歧义数据均被拒绝。 `/Encrypt`、`/ObjStm` 与 `/XRef` 无论出现在哪里都会被拒绝,名称转义后也不例外; diff --git "a/docs/\346\226\207\344\273\266\346\211\271\351\207\217\346\223\215\344\275\234.md" "b/docs/\346\226\207\344\273\266\346\211\271\351\207\217\346\223\215\344\275\234.md" index 828bae95..a94aaeb3 100644 --- "a/docs/\346\226\207\344\273\266\346\211\271\351\207\217\346\223\215\344\275\234.md" +++ "b/docs/\346\226\207\344\273\266\346\211\271\351\207\217\346\223\215\344\275\234.md" @@ -43,8 +43,10 @@ Native 输出文件标题、物理行号、逐文件错误与续读提示;程 `ops_edit_file` 对**单个** UTF-8 文件原子应用 1–100 段修改。参数包括 `path`、读取/stat 得到的准确 SHA-256 `expectedRevision` 与 `edits: [{oldText,newText}]`。每个非空 `oldText` 必须在同一原始字节中 -唯一出现;重叠、歧义或版本过期会在发布前拒绝全部修改。空 `newText` 表示删除。CRLF/BOM 需显式 -保留。`dryRun: true` 仅预览相同回执;只读策略仍拒绝写入工具。这不是跨文件事务。 +唯一出现。若原始匹配失败,会尝试一次可逆 Native 渲染解码,同时解码 `oldText` 与 `newText`; +原始匹配不会解码任何字段,回退回执以 `renderedMatches` 计数。重叠、歧义或版本过期会在发布前拒绝全部修改。 +空 `newText` 表示删除。CRLF/BOM 需显式保留。`dryRun: true` 仅预览相同回执;只读策略仍拒绝写入工具。 +这不是跨文件事务。 它共享 `ops_edit_file` 的 no-follow、CAS 与平台元数据保护,返回包含版本、修改计数、字节/行数及 警告的短回执,不回显替换内容。重试前必须检查 `error.effectState`;该工具不会标记为并发安全。 diff --git "a/docs/\347\264\247\345\207\221\346\226\207\344\273\266\350\214\203\345\233\264.md" "b/docs/\347\264\247\345\207\221\346\226\207\344\273\266\350\214\203\345\233\264.md" index b263d2ab..b658150e 100644 --- "a/docs/\347\264\247\345\207\221\346\226\207\344\273\266\350\214\203\345\233\264.md" +++ "b/docs/\347\264\247\345\207\221\346\226\207\344\273\266\350\214\203\345\233\264.md" @@ -12,7 +12,7 @@ English: [Compact Native file ranges](native-ranges.md). [revision=cfd1bc81432fd6b88748b827f46bb4242db17cf8655cf4ea90f3ba0e62065355; more lines; nextOffset=3] ``` -`N|` 后紧跟原始行内容,包含原有缩进。正文使用可逆 JSON 转义:反斜杠写成双反斜杠,控制字符及 Unicode 换行分隔符被转义,引号保持原样。例如真实制表符显示为 `\t`,而反斜杠加字母 t 的两个源字符显示为 `\\t`。紧凑范围和普通行号正文共用此编码,防止伪造额外行号或文件标题;LF/CRLF 物理行规则与 worker 保持一致。 +`N|` 后紧跟原始行内容,包含原有缩进。正文使用可逆 JSON 转义:反斜杠写成双反斜杠,控制字符及 Unicode 换行分隔符被转义,引号保持原样。例如真实制表符显示为 `\t`,而反斜杠加字母 t 的两个源字符显示为 `\\t`。紧凑范围和普通行号正文共用此编码,防止伪造额外行号或文件标题;LF/CRLF 物理行规则与 worker 保持一致。`ops_edit_file` 始终先匹配字面字节;只有原始匹配失败后才可能解码此渲染形式的 `oldText` 与 `newText`,并在回执中返回 `renderedMatches`。原始匹配与 `write_file` 内容不会被解码。 `more lines` 表示文件尚未读到末尾,按 `nextOffset` 继续;`end of file; complete=true` 表示本范围已经到达文件末尾。汇总的 `requested ranges returned` 仅证明所请求的范围正文全部返回,不代表读取了所有完整文件。完整版本散列、总行数和每个文件的续读位置继续保留;字节大小及重复计数仍在规范数据和 PTC 返回值中。逐文件版本回执省去重复的 `Notice: ` 前缀与末尾句点;汇总 `Notice` 以及版本、续读和完整性事实均保留。