From fe0c065d3a68ee338a07c820e823c72a42e1acc6 Mon Sep 17 00:00:00 2001 From: Cedric-2714 Date: Fri, 23 Oct 2026 19:09:28 -0400 Subject: [PATCH 1/6] feat(workspace)!: read, grep, edit and edit_many answer as the text the model reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The model read each of these four results as one line of JSON: a page's numbered lines escaped inside a `content` string, grep's matches inside nested objects, a receipt's changed region inside `changed`. Each now answers with its text as data (one text block), and the same fields it had beside it as `structured`, which a code_execution script reads as structuredContent (#262's rule; #698 sends only a string data as text, so it didn't reach these). The text (result-text.ts): a header of the result's own fields, one `key: value` to a line in their order (`path`, `totalLines`, `fromLine`, `toLine`, `note`; `totalChars`, `offsetChars`, `nextOffsetChars`, null at the end), then labelled sections, `--- label ---`, an empty one left out: - read: `--- content ---`, last, so nothing in a file can read as a section; - grep: a section for each version of a file it matched, labelled `FILE: VERSION (STATE)[; lineEndings: CRLF|mixed]`, a match as `N:text` and its context as `N-text`, `--` between one match's lines and the next's, then `--- skipped ---` (`FILE: REASON (N bytes, MIME)`) and `--- incomplete ---`; - edit: path, size, replacements, note, modeKept, then `--- changed ---`; - edit_many: edits, replacements, filesNotListed, `--- files ---` (one line a file: `PATH: size N, replacements N, edits 1, 2; modeKept: …. NOTE`), then `--- changed ---`. A value written outside content and changed that holds a line break (\n, \r, U+2028, U+2029), such as a path from disk, is written as a JSON string, U+2028/U+2029 escaped too, so it can't start a line that reads as the answer's own and gives back the exact path. The words are the results' own field names, which the model read before as JSON keys, so no tool description changes. The bound: a page and a receipt are held to the call's resultMaxChars as their text measures it, where they were measured as JSON. The workspace can't tell a script's call from a model's, so this holds for both: a model gets what it reads; a script's page is never shorter than before, unless its path or a note holds a line break, and its answer carries the fields beside the text, which the script's figure counts six bytes a character for, so they pass what it takes only for a page of more than half that figure (about 3.7 million characters at the default 128 MiB of script memory) holding control characters. jsonLength and jsonCharLength had no other user and go. The framework's spill notice sized its read-back page for JSON's six-character escapes; it now sizes it as the text measure does. #465's pending changelog gives two path lengths at which a page at 1,000 characters passes the limit, measured as JSON. Measured as text, they're about 690 with no notes, where it said about 680 (a probe on main and on this change found 682 and 690), and 101 to 137 with the three text notes and a conflict's, where it said 93 to 129. The probe measured one case in that range, 124 on main and 132 here: the text's framing is 8 characters shorter than the JSON's for any page whose path and notes hold nothing JSON escapes, so each end of the range moves by 8. Tests: workspace-result-text.test.ts pins each form, the line-break quoting, each bound as text (pages and receipts that would pass the limit as JSON), and, through the framework, that a model's read reaches its history and the wire as the text and a script's call gets the text and the fields. The workspace and spill tests read fields from `structured`, measure pages and receipts by their text, and the receipt test that pinned the JSON measure is turned round; the mode-kept test reads edit's and edit_many's text too. Co-Authored-By: Claude Opus 5.5 --- .../cedric-f55-workspace-text.breaking.md | 43 +++ changelog.d/feat-workspace-read.changed.md | 2 +- src/framework.ts | 11 +- src/modules/workspace/edit-text.ts | 82 ++-- src/modules/workspace/index.ts | 55 ++- src/modules/workspace/read-page.ts | 85 +++-- src/modules/workspace/result-text.ts | 104 ++++++ src/modules/workspace/text.ts | 34 +- test/tool-result-cap-fit.test.ts | 6 +- test/tool-result-spill.test.ts | 9 +- test/workspace-direct-write.test.ts | 2 +- test/workspace-direct.test.ts | 24 +- test/workspace-edits.test.ts | 77 ++-- test/workspace-file-tools.test.ts | 90 ++--- test/workspace-glob-mount-prefix.test.ts | 2 +- test/workspace-glob-unclosed-brace.test.ts | 4 +- test/workspace-grep-filter.test.ts | 6 +- test/workspace-include.test.ts | 8 +- test/workspace-materialize-freshness.test.ts | 2 +- test/workspace-nested-mount-roots.test.ts | 16 +- test/workspace-parent-link.test.ts | 2 +- test/workspace-path-spelling.test.ts | 44 +-- test/workspace-read-text.test.ts | 153 ++++---- test/workspace-result-text.test.ts | 351 ++++++++++++++++++ test/workspace-text.test.ts | 13 +- test/workspace-three-way.test.ts | 26 +- 26 files changed, 892 insertions(+), 359 deletions(-) create mode 100644 changelog.d/cedric-f55-workspace-text.breaking.md create mode 100644 src/modules/workspace/result-text.ts create mode 100644 test/workspace-result-text.test.ts diff --git a/changelog.d/cedric-f55-workspace-text.breaking.md b/changelog.d/cedric-f55-workspace-text.breaking.md new file mode 100644 index 000000000..8192d0c03 --- /dev/null +++ b/changelog.d/cedric-f55-workspace-text.breaking.md @@ -0,0 +1,43 @@ +- **Scripts, hosts and modules that read the workspace's `read`, `grep`, + `edit` or `edit_many` results:** each now answers as the text the model + reads, with its fields beside it in `structured`. The model read each as + one line of JSON, a page's lines escaped inside it. It now reads a header + of the result's fields, one `key: value` to a line (`path: …`, + `totalLines: …`, `nextOffsetChars: null`), then labelled sections: a + page's `--- content ---`, grep's matches by file and version (`N:` for a + match, `N-` for a line of its context), what grep skipped, and a receipt's + `--- files ---` and `--- changed ---`. A value holding a line break, such + as a path, is written as a JSON string. A control character in a file + reaches the model as itself, where JSON wrote an escape such as `\u001b`. + + **Who needs to act:** + - a `code_execution` script that parses one of these results' text as + JSON (`JSON.parse(result.content[0].text)`): read + `result.structuredContent`, which holds the fields the JSON held; + - a host or module that reads these results' fields from `result.data`, + through `executeToolCall` or `ctx.callTool`: read `result.structured`. + connectome-host reads either since 0.11.0 + (anima-research/connectome-host#324). + + **What a page holds:** a page is held to the call's result limit as its + text measures it, where it was measured as JSON, and so is a receipt. + - A model's page holds more of a file whose lines JSON escapes: at the + default 24,000 characters, a page of this repository's + `src/modules/workspace/index.ts` now ends at line 475, where it ended at + line 454. A spilled result's notice sizes the read-back page it suggests + the same way, so below a cap of about 2,300 characters it suggests a + longer one. + - A script's page is never shorter than before, since a page's text is + shorter than its JSON. A script's answer carries the fields beside the + text, which the limit doesn't count. Together they can pass what a + script takes only on a page of more than about 3.7 million characters + that holds control characters, which JSON writes as six bytes each where + no other character takes more than three. Such a page answers + `ResultTooLarge`, which cuts nothing, and the script reads a smaller + page. + + **Unchanged:** the tools' descriptions and inputs; the words of their + notes; their errors, which were text already; the results of `write`, + `delete`, `ls`, `glob`, `status`, `materialize`, `sync` and the other + workspace tools, whose fields stay in `data`; and a page for a call made + in process without a result limit, which holds to its line limit alone. diff --git a/changelog.d/feat-workspace-read.changed.md b/changelog.d/feat-workspace-read.changed.md index c94feec88..137233c84 100644 --- a/changelog.d/feat-workspace-read.changed.md +++ b/changelog.d/feat-workspace-read.changed.md @@ -1,6 +1,6 @@ - The workspace's `read` shows a file's text as it is, and says so. A line page shows a CRLF file's lines without the CR before each line feed, and leaves out a BOM. A line whose own text ends in a CR shows its line end's CR too, since `edit` reads a CR before a line feed as part of the line end, so text copied from the line still matches it. Its note says when the file has CRLF ends or a BOM, counting a mix of CRLF and LF ends and the lines that show both CRs. It also says when the file isn't valid UTF-8: each invalid byte sequence shows as U+FFFD, `edit` refuses the file, and `write` replaces it whole. Before, each line of a CRLF file ended in a CR, a BOM came first in the first line as U+FEFF, and invalid bytes showed as U+FFFD with no note. A character page still reads the stored text as it is, BOM and CRs included. - On a page for a call that carries a `resultMaxChars`, such as a model's or a running script's, a line over 2,000 characters is cut, with a marker that gives the `offsetChars` and `limitChars` that read its rest. Before, a line page showed every line whole, however long. -- A `read` page stops before its result would pass `resultMaxChars`, the most of it that can reach its caller whole. For a model's call that's its agent's inline cap, 24,000 characters by default, within which a result is stored whole unless its round's results together pass the readers' bound; for a running script's call it's what the script's memory takes, 7,456,527 characters at the default 128 MiB. Its note says where to continue: the `offset` after its last line, or for a character page the `offsetChars` it also gives as `nextOffsetChars`, with the `limitChars` left of what it was asked for, so a reader that follows the notes ends where it asked to. Where even a page's first line would pass the limit, that line is cut shorter, at the most the page has room for. Where a page's last line is cut, its note names the `offsetChars` where that line's rest starts and the `limitChars` that reads it to the line's end, before the next line's `offset` if the file goes on. A page always moves on, so where its own fields (the path and its notes) and the least it can show pass a small limit, it passes the limit. At 1,000 characters, a page with no notes does that only for a path of about 680 characters or more. Its notes take room from that: with all three notes on a file's text and a conflict's, a path of 93 to 129 characters does it, and a note that names a symlink's target, as one saying disk couldn't be checked can, takes as much room as the target is long. Before, a page of up to 2,000 lines, or of `limitChars` characters, went out whole, and for a result that passed the limit the framework kept only its start, cut mid-JSON, writing the whole of it to a workspace file where the agent has one. A caller that pages 2,000 lines at a time and starts the next page at line 2,001 now starts where the note says. A running `code_execution` script's call has that larger limit, so its line pages still stop at the line limit, 2,000 by default, and its character pages at `limitChars`, unless a page would pass that. Its line pages cut a line over 2,000 characters and leave out CRs and a BOM, as a model's do, and a character page gives it the stored text exactly. A call made in process, through `handleToolCall`, `executeToolCall` or `ModuleContext.callTool`, carries no `resultMaxChars` unless its caller sets one. Its line pages stop at the line limit alone and show every line whole, as they did before, though now without the CRs and the BOM. +- A `read` page stops before its result would pass `resultMaxChars`, the most of it that can reach its caller whole. For a model's call that's its agent's inline cap, 24,000 characters by default, within which a result is stored whole unless its round's results together pass the readers' bound; for a running script's call it's what the script's memory takes, 7,456,527 characters at the default 128 MiB. Its note says where to continue: the `offset` after its last line, or for a character page the `offsetChars` it also gives as `nextOffsetChars`, with the `limitChars` left of what it was asked for, so a reader that follows the notes ends where it asked to. Where even a page's first line would pass the limit, that line is cut shorter, at the most the page has room for. Where a page's last line is cut, its note names the `offsetChars` where that line's rest starts and the `limitChars` that reads it to the line's end, before the next line's `offset` if the file goes on. A page always moves on, so where its own fields (the path and its notes) and the least it can show pass a small limit, it passes the limit. At 1,000 characters, a page with no notes does that only for a path of about 690 characters or more. Its notes take room from that: with all three notes on a file's text and a conflict's, a path of 101 to 137 characters does it, and a note that names a symlink's target, as one saying disk couldn't be checked can, takes as much room as the target is long. Before, a page of up to 2,000 lines, or of `limitChars` characters, went out whole, and for a result that passed the limit the framework kept only its start, cut mid-JSON, writing the whole of it to a workspace file where the agent has one. A caller that pages 2,000 lines at a time and starts the next page at line 2,001 now starts where the note says. A running `code_execution` script's call has that larger limit, so its line pages still stop at the line limit, 2,000 by default, and its character pages at `limitChars`, unless a page would pass that. Its line pages cut a line over 2,000 characters and leave out CRs and a BOM, as a model's do, and a character page gives it the stored text exactly. A call made in process, through `handleToolCall`, `executeToolCall` or `ModuleContext.callTool`, carries no `resultMaxChars` unless its caller sets one. Its line pages stop at the line limit alone and show every line whole, as they did before, though now without the CRs and the BOM. - The workspace's `grep` leaves out a CRLF end's CR, for matching as well as display, as `read` does, so `foo$` finds a CRLF file's line too, as it does under ripgrep's `--crlf`. A match's text and context are its lines as `read` shows them, but each line whole, where a `read` page for a call with a result limit cuts a line over 2,000 characters. A BOM isn't part of the first line, and a file with CRLF ends says so in its result's `lineEndings` (`CRLF`, or `mixed`). Before, a CRLF file's lines ended in a CR, so `foo$` missed them and a match's text copied into an edit carried the CR. A pattern written to find the CR of a CRLF line end no longer matches, except on a line whose own text ends in a CR, which keeps its line end's CR as `read` shows it. When a pattern names a CR, as a literal one or as `\r`, `\x0d`, `\u000d`, `\cM` or `\015`, and a file it searched has CRLF ends, the result's `note` says how many files searched have them and names them, each as `CRLF` or `mixed`, up to 20 and then how many more. It also says that a character `read` shows every CR in the workspace's copy, or in disk's file on a direct mount where the host has one. A pattern that names U+FEFF, over a file that starts with a BOM, gets a note that names the files that start with one and says a BOM isn't part of the first line. So a `grep` for `\r` or `^\uFEFF` still finds which of the files it searches have CRLF ends or a BOM. A CR that doesn't come before a line feed is still part of its line. - The workspace's `read` first takes in a change on disk to a file the workspace holds, as a listing and an `edit` do, and shows that text. Its note says so when disk's copy had changed since the workspace last agreed with it. It also says when disk's copy couldn't be checked, or disk has none and nothing records whether it was deleted there, so it shows the workspace's copy; where the check gave up because the selected branch kept changing, it says to read again. A file disk deleted answers that disk deleted it and the deletion was taken in, and a file disk made binary or too large answers that it's on disk but not stored. Where the path's stored copy is a draft and disk changed or deleted it too, the read shows the draft, and its note names the conflict and the ways to settle it, a sync, or a forced materialize on a mount that isn't read-only: for a deletion, a sync of the path takes it and removes the workspace's copy. Before, a read of a path the workspace held didn't look at disk, so it could show a copy another writer had since changed; a path it didn't hold was observed, as now. - A direct mount's `read` and `grep` show a file's text as a store-backed mount's do, with the same pages, cut lines, notes and `lineEndings`. Only the disk check is a store-backed mount's alone, since a direct mount reads disk as it is and keeps no copy of its own. diff --git a/src/framework.ts b/src/framework.ts index 877428fec..f0f9dcb45 100644 --- a/src/framework.ts +++ b/src/framework.ts @@ -21876,11 +21876,12 @@ export class AgentFramework { * the inline cap would show more: where a cap at the readers' bound would * keep a longer head, or the whole result. */ private spillPageAdvice(path: string, cap: number, canRaise: boolean): string { - // JSON serialization can expand each read-back code unit to six - // chars (\\u0000). Leave room for the page's metadata and a possible - // extra code unit at a surrogate-pair boundary. - const pageOverhead = JSON.stringify(path).length + 256; - const readLimit = Math.max(1, Math.min(2000, Math.floor((cap - pageOverhead) / 6) - 1)); + // A page is held to the cap as its text measures it, each code unit one + // character (the workspace's read-page.ts). Leave room for the page's + // fields and notes, and a possible extra code unit at a surrogate-pair + // boundary. + const pageOverhead = path.length + 256; + const readLimit = Math.max(1, Math.min(2000, cap - pageOverhead - 1)); return `Page with workspace--read ${JSON.stringify({ path, offsetChars: 0, limitChars: readLimit })}; ` + 'use nextOffsetChars as offsetChars until null, keeping limitChars.' + (canRaise ? ' Or raise the inline cap via agent_settings update tool_result_inline_max_chars.' : ''); diff --git a/src/modules/workspace/edit-text.ts b/src/modules/workspace/edit-text.ts index 4516c25c1..b4467017a 100644 --- a/src/modules/workspace/edit-text.ts +++ b/src/modules/workspace/edit-text.ts @@ -11,8 +11,10 @@ * and EditMany tools' answers. */ +import type { ToolResult } from '../../types/events.js'; import { type Change, type FilePlan, type Leaves, type MatchProblem, SHOWN_MATCHES } from './edit-plan.js'; -import { MAX_LINE_CHARS, jsonLength, numberedLine, safeCut } from './text.js'; +import { answerText, fieldLine, sectionLead, sectionText, shownValue, textAnswer } from './result-text.js'; +import { MAX_LINE_CHARS, numberedLine, safeCut } from './text.js'; /** The tool a call came through, which words its refusals and receipt. */ export type EditTool = 'edit' | 'edit_many'; @@ -363,7 +365,7 @@ function excerptLine(line: string, n: number, columns: number[]): string { /** * Each edit's first change in a file, as numbered regions merged where they - * meet, while the budget lasts, counted as the answer's JSON holds them; past + * meet, while the budget lasts, counted as the answer's text holds them; past * it, the regions are named by their lines. */ export function shownRegions(plan: FilePlan, budget: { left: number }, maxChars: number): string[] { @@ -409,7 +411,7 @@ export function shownRegions(plan: FilePlan, budget: { left: number }, maxChars: // region of millions of lines, which is named by its lines, isn't built // whole first. The line before it ("…" between regions, or the line end) // counts too, and each line feed between its lines. - let cost = out.length ? jsonLength('\n…\n') : jsonLength('\n'); + let cost = out.length ? '\n…\n'.length : '\n'.length; const lines: string[] = []; let fits = !omitted.length; for (let lineStart = region.start, n = region.from; fits && lineStart < region.end; n++) { @@ -420,7 +422,7 @@ export function shownRegions(plan: FilePlan, budget: { left: number }, maxChars: // changes start on it, a stretch from shortly before each. const columns = text.length > MAX_LINE_CHARS ? firsts.filter((first) => first.line === n).map((first) => first.at - lineStart) : []; const numberedText = columns.length ? excerptLine(text, n, columns) : numberedLine(text, n); - cost += jsonLength(numberedText) + (lines.length ? jsonLength('\n') : 0); + cost += numberedText.length + (lines.length ? 1 : 0); if (cost > budget.left) fits = false; else lines.push(numberedText); lineStart = lineEnd + 1; @@ -520,24 +522,50 @@ function receiptReserve(tool: EditTool, files: StoredFile[], maxChars: number): const listed = Math.min(SHOWN_MATCHES, plan.edits.length); const range = `${last}–${last}`; const note = regionsLeftOut(maxChars, Array.from({ length: listed }, () => range), plan.edits.length - listed, false); - regions = Math.max(regions, jsonLength('\n') + jsonLength(note)); + regions = Math.max(regions, 1 + note.length); } if (tool === 'edit') return regions; const most = files.length; - return regions + jsonLength('\n\n') + jsonLength(filesLeftOut(maxChars, most, { notListed: most, unshown: most, notes: most, edits: most })!); + return regions + 2 + filesLeftOut(maxChars, most, { notListed: most, unshown: most, notes: most, edits: most })!.length; +} + +/** A file edit_many lists, as its receipt's fields hold it. */ +interface ListedFile { + path: string; + size: number; + replacements: number; + modeKept?: string; + edits?: number[]; + note?: string; +} + +/** edit_many's line for a file it lists: its path, size and replacements, and then the edits, the mode kept and the notes it has room for. */ +function listedFileLine(file: ListedFile): string { + return `${shownValue(file.path)}: size ${file.size}, replacements ${file.replacements}` + + (file.edits !== undefined ? `, edits ${file.edits.join(', ')}` : '') + + (file.modeKept !== undefined ? `; modeKept: ${shownValue(file.modeKept)}` : '') + + (file.note !== undefined ? `. ${shownValue(file.note)}` : ''); +} + +/** edit_many's receipt's text before its changed regions: its counts, and the files it lists. */ +function manyHeadText(data: { edits: number; replacements: number; files: ListedFile[]; filesNotListed?: number }): string { + const header = [fieldLine('edits', data.edits), fieldLine('replacements', data.replacements)]; + if (data.filesNotListed !== undefined) header.push(fieldLine('filesNotListed', data.filesNotListed)); + return answerText(header, [sectionText('files', data.files.map(listedFileLine).join('\n'))]); } /** * The receipt of edits stored: each file's size, replacements and notes, and * each edit's first changed region, numbered as read numbers lines, held to - * `maxChars` as the answer's JSON holds it. Its counts and notes come first; - * the regions take what is left, and past it a file's regions are named by - * their lines. edit_many lists its files while they fit, a file whose notes - * don't fit without them, or without its edits' numbers too, and counts the - * rest. edit's receipt, of one file, keeps its counts and notes and shows no - * region where those alone pass `maxChars`. + * `maxChars` as the answer's text holds it (result-text.ts). Its counts and + * notes come first; the regions take what is left, and past it a file's + * regions are named by their lines. edit_many lists its files while they + * fit, a file whose notes don't fit without them, or without its edits' + * numbers too, and counts the rest. edit's receipt, of one file, keeps its + * counts and notes and shows no region where those alone pass `maxChars`. + * The model reads the text; a script gets the receipt's fields beside it. */ -export function editReceipt(tool: EditTool, files: StoredFile[], editCount: number, maxChars: number): Record { +export function editReceipt(tool: EditTool, files: StoredFile[], editCount: number, maxChars: number): ToolResult { const reserve = receiptReserve(tool, files, maxChars); const notesOf = ({ plan, notes }: StoredFile): string[] => { const lineEnds = lineEndNote(plan); @@ -557,28 +585,28 @@ export function editReceipt(tool: EditTool, files: StoredFile[], editCount: numb if (tool === 'edit') { const file = files[0]!; const notes = notesOf(file); - const data: Record = { + const head = { path: file.path, size: file.plan.bytes.length, replacements: file.plan.replacements, ...(notes.length ? { note: notes.join(' ') } : {}), ...(file.modeKept ? { modeKept: file.modeKept } : {}), - changed: '', }; - const budget = { left: maxChars - JSON.stringify(data).length - reserve }; - data.changed = shownRegions(file.plan, budget, maxChars).join('\n'); - return data; + const header = Object.entries(head).map(([key, value]) => fieldLine(key, value)); + const budget = { left: maxChars - answerText(header).length - sectionLead('changed') - reserve }; + const changed = shownRegions(file.plan, budget, maxChars).join('\n'); + return textAnswer(answerText(header, [sectionText('changed', changed)]), { ...head, changed }); } - const listed: Array> = []; - const data: Record = { + const listed: ListedFile[] = []; + const data: { edits: number; replacements: number; files: ListedFile[]; filesNotListed?: number; changed: string } = { edits: editCount, replacements: files.reduce((sum, file) => sum + file.plan.replacements, 0), files: listed, changed: '', }; - const fits = (entry: Record) => { + const fits = (entry: ListedFile) => { listed.push(entry); - const fit = JSON.stringify(data).length <= maxChars - reserve; + const fit = manyHeadText(data).length + sectionLead('changed') <= maxChars - reserve; listed.pop(); return fit; }; @@ -587,7 +615,7 @@ export function editReceipt(tool: EditTool, files: StoredFile[], editCount: numb let notListed = 0; for (const [i, file] of files.entries()) { const notes = notesOf(file); - const bare = { + const bare: ListedFile = { path: file.path, size: file.plan.bytes.length, replacements: file.plan.replacements, @@ -610,22 +638,22 @@ export function editReceipt(tool: EditTool, files: StoredFile[], editCount: numb if (notes.length && !('note' in entry)) notesLeftOut++; if (!('edits' in entry)) editsLeftOut++; } - const budget = { left: maxChars - JSON.stringify(data).length - reserve }; + const budget = { left: maxChars - manyHeadText(data).length - sectionLead('changed') - reserve }; const sections: string[] = []; let unshown = 0; for (const file of files.slice(0, listed.length)) { - const heading = jsonLength(`${file.path}\n`) + (sections.length ? jsonLength('\n\n') : 0); + const heading = shownValue(file.path).length + 1 + (sections.length ? 2 : 0); if (heading >= budget.left) { unshown++; continue; } budget.left -= heading; - sections.push(`${file.path}\n${shownRegions(file.plan, budget, maxChars).join('\n')}`); + sections.push(`${shownValue(file.path)}\n${shownRegions(file.plan, budget, maxChars).join('\n')}`); } const last = filesLeftOut(maxChars, files.length, { notListed, unshown, notes: notesLeftOut, edits: editsLeftOut }); if (last !== null) sections.push(last); data.changed = sections.join('\n\n'); - return data; + return textAnswer(answerText([manyHeadText(data)], [sectionText('changed', data.changed)]), data); } const capitalized = (value: string) => value.charAt(0).toUpperCase() + value.slice(1); diff --git a/src/modules/workspace/index.ts b/src/modules/workspace/index.ts index 82a82901d..bffc0b3ab 100644 --- a/src/modules/workspace/index.ts +++ b/src/modules/workspace/index.ts @@ -47,6 +47,7 @@ import { type EditSpec, type FilePlan, countBudget, decodeForEdit, hintBudget, p import { type EditProblem, type EditTool, cutLine, editList, editReceipt, fitted, refusalText } from './edit-text.js'; import { type FileText, fileText } from './text.js'; import { DEFAULT_READ_LINE_LIMIT, characterPage, linePage } from './read-page.js'; +import { type GrepAnswer, grepText, textAnswer } from './result-text.js'; import { type MountRuntime, type PassOptions, @@ -4404,15 +4405,12 @@ export class WorkspaceModule implements Module { change: { kind: 'set' as const, blobHash: store.storeBlob(plan.bytes, 'text/plain'), size: plan.bytes.length, mode }, })), () => { committed = true; }); if (refused.size > 0) return this.editsStoredButUnfinished(tool, [...targets.values()], applied, conflicts, { refused, modeKept }, maxChars); - return { - success: true, - data: editReceipt(tool, plans.map(({ target, plan, backOnDisk, conflictNote }) => ({ - path: target.path, - plan, - notes: [applied.get(target) ?? null, backOnDisk, conflictNote].filter((note): note is string => note !== null), - modeKept: modeKept.get(target.relativePath) ?? null, - })), given.length, maxChars), - }; + return editReceipt(tool, plans.map(({ target, plan, backOnDisk, conflictNote }) => ({ + path: target.path, + plan, + notes: [applied.get(target) ?? null, backOnDisk, conflictNote].filter((note): note is string => note !== null), + modeKept: modeKept.get(target.relativePath) ?? null, + })), given.length, maxChars); }); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -4795,15 +4793,12 @@ export class WorkspaceModule implements Module { } } this.recordDirectWrite(mount, agentName); - return { - success: true, - data: editReceipt(tool, staged.map(({ target, staged: file }) => ({ - path: target.path, - plan: plans.find((p) => p.target === target)!.plan, - notes: [reread(target, 'applied to'), linksNote(target.path, file, tool)].filter((note): note is string => note !== null), - modeKept: null, - })), editCount, maxChars), - }; + return editReceipt(tool, staged.map(({ target, staged: file }) => ({ + path: target.path, + plan: plans.find((p) => p.target === target)!.plan, + notes: [reread(target, 'applied to'), linksNote(target.path, file, tool)].filter((note): note is string => note !== null), + modeKept: null, + })), editCount, maxChars); } /** @@ -5432,20 +5427,18 @@ export class WorkspaceModule implements Module { incomplete.push(...found.incomplete); } - return { - success: true, - data: { - pattern: input.pattern, - results, - totalMatches: results.reduce((sum, r) => sum + r.matches.length, 0), - // A pattern written for a CR can't find a CRLF end's, nor one written - // for U+FEFF a BOM, which lines as read shows them leave out, so the - // answer says why it didn't, and names the files that have them. - ...leftOutNote(input.pattern, crlfFiles, bomFiles, this.anyRunsDirect()), - ...(skipped.length > 0 ? { skipped } : {}), - ...(incomplete.length > 0 ? { incomplete } : {}), - }, + const answer: GrepAnswer = { + pattern: input.pattern, + results, + totalMatches: results.reduce((sum, r) => sum + r.matches.length, 0), + // A pattern written for a CR can't find a CRLF end's, nor one written + // for U+FEFF a BOM, which lines as read shows them leave out, so the + // answer says why it didn't, and names the files that have them. + ...leftOutNote(input.pattern, crlfFiles, bomFiles, this.anyRunsDirect()), + ...(skipped.length > 0 ? { skipped } : {}), + ...(incomplete.length > 0 ? { incomplete } : {}), }; + return textAnswer(grepText(answer, contextBefore), { ...answer }); } /** diff --git a/src/modules/workspace/read-page.ts b/src/modules/workspace/read-page.ts index 6d880cc9a..2595ae318 100644 --- a/src/modules/workspace/read-page.ts +++ b/src/modules/workspace/read-page.ts @@ -1,12 +1,17 @@ import type { ToolResult } from '../../types/events.js'; -import { type FileText, MAX_LINE_CHARS, jsonCharLength, jsonLength, numberedLine, safeCut } from './text.js'; +import { answerText, fieldLine, sectionLead, sectionText, textAnswer } from './result-text.js'; +import { type FileText, MAX_LINE_CHARS, numberedLine, safeCut } from './text.js'; /** - * One page of a read, held to the result limit: the framework gives a - * success's data as JSON, and what passes the caller's limit doesn't reach it - * whole. A model's result spills into a file, cut mid-JSON, and a script gets - * ResultTooLarge in its place. So a page stops before it would pass the limit, - * and its note says where to continue. + * One page of a read, held to the result limit as its text measures it: a + * model reads the page's text (result-text.ts), and what passes the caller's + * limit doesn't reach it whole. A model's result spills into a file, cut, and + * a script gets ResultTooLarge in its place. So a page stops before its text + * would pass the limit, and its note says where to continue. A script's + * answer carries the page's fields beside its text, which the limit doesn't + * count but leaves room for, since it counts each character at the six bytes + * it takes at most in the script's answer: the two pass it only together, on + * a page of millions of characters, most of them control characters. */ /** How many lines a read shows when it isn't given a limit. */ @@ -36,10 +41,27 @@ const joined = (notes: Array): string | undefined => { return kept.length > 0 ? kept.join(' ') : undefined; }; +/** A page's fields: a line page's or a character page's, and its content, numbered lines or raw text. */ +type Page = { path: string; content: string; note?: string } & Record; + +/** A page's text: its fields, one to a line in their order, and then its content under its label. */ +function pageText(page: Page): string { + const header = Object.entries(page) + .filter((entry): entry is [string, string | number | null] => entry[0] !== 'content' && entry[1] !== undefined) + .map(([key, value]) => fieldLine(key, value)); + return answerText(header, [sectionText('content', page.content)]); +} + +/** A page as read answers it: its text for the model, and its fields for a script. */ +const pageAnswer = (page: Page): ToolResult => textAnswer(pageText(page), page); + +/** What content adds to its page's text: its label's line with the line feed before it, and its own characters. */ +const contentCost = (chars: number): number => (chars > 0 ? sectionLead('content') + chars : 0); + /** * Lines from `offset` (1-based), at most `limit` of them, each numbered and a * long one cut with the offsetChars and limitChars that read the rest. The - * page is the longest that fits within `maxChars` as JSON; where none fits + * page is the longest whose text fits within `maxChars`; where none fits * with its first line as a page shows it, that line is cut shorter, at the * most the page has room for. Where the page's last line is cut, its note * names where that line's rest starts and how long it is, since reading on in @@ -62,7 +84,7 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu first > 0 ? `Line ${first + 1} is past the end: the file has ${total} line${total === 1 ? '' : 's'}.` : null, ...notes, ]); - return { success: true, data: { path, totalLines: total, ...(note !== undefined ? { note } : {}), content: '' } }; + return pageAnswer({ path, totalLines: total, ...(note !== undefined ? { note } : {}), content: '' }); } const last = Math.min(first + limit, total); const shown = (i: number, maxLineChars = MAX_LINE_CHARS) => @@ -81,12 +103,12 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu const budgetNote = (to: number, rest: number | null) => `Stopped at line ${to} to stay within the ${maxChars}-character result limit (file has ${total}). ` + (rest === null ? `Continue with offset=${to + 1}.` : `${cutLine(to, rest)}, then continue with offset=${to + 1}.`); - const page = (to: number, content: string, note: string | undefined) => ({ + const page = (to: number, content: string, note: string | undefined): Page => ({ path, totalLines: total, fromLine: first + 1, toLine: to, ...(note !== undefined ? { note } : {}), content, }); if (maxChars === undefined) { const whole = lines.slice(first, last).map((line, k) => numberedLine(line, first + k + 1, Infinity)).join('\n'); - return { success: true, data: page(last, whole, joined([last < total ? limitNote(last, null) : null, ...notes])) }; + return pageAnswer(page(last, whole, joined([last < total ? limitNote(last, null) : null, ...notes]))); } // The note of the page that ends at line `to`, its last line shown at most // lastMax long: the page that ends where it was asked to, at the file's end @@ -96,10 +118,10 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu const endNote = to < total ? limitNote(to, rest) : rest === null ? null : `${cutLine(to, rest)}.`; return joined([to === last ? endNote : budgetNote(to, rest), ...notes]); }; - // A page's length as JSON: its fields' and note's, measured with no content, - // and its lines', which JSON writes inside the content's quotes. + // A page's text's length: its fields' and note's, measured with no content, + // and its content's, under the content's label. const lengthAt = (to: number, contentChars: number, note: string | undefined) => - JSON.stringify(page(to, '', note)).length + contentChars; + pageText(page(to, '', note)).length + contentCost(contentChars); // The page is the longest that fits. Its length isn't monotone in its // lines: the page that ends the file carries no stop note, the one at the // line limit a shorter one than a stop, and one whose last line is cut a @@ -111,14 +133,14 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu for (let i = first; i < last; i++) { const line = shown(i); shownLines.push(line); - // Each line after the first comes after a line feed, which JSON writes as two characters. - contentChars += jsonLength(line) + (i > first ? 2 : 0); + // Each line after the first comes after a line feed. + contentChars += line.length + (i > first ? 1 : 0); if (lengthAt(i + 1, contentChars, noteAt(i + 1)) <= maxChars) best = shownLines.length; else if (lengthAt(i + 1, contentChars, joined(notes)) > maxChars) break; } if (best > 0) { shownLines.length = best; - return { success: true, data: page(first + best, shownLines.join('\n'), noteAt(first + best)) }; + return pageAnswer(page(first + best, shownLines.join('\n'), noteAt(first + best))); } // No page fits with its first line as a page shows it: the loop tried that // page first, since a read's limit is at least 1, and it is pageWith(top). @@ -134,7 +156,7 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu // that fits is the one the search finds or, past a dip, at most two cuts // further, four code units at most, and those are tried too. const pageWith = (chars: number) => page(first + 1, shown(first, chars), noteAt(first + 1, chars)); - const fitsAt = (chars: number) => JSON.stringify(pageWith(chars)).length <= maxChars; + const fitsAt = (chars: number) => pageText(pageWith(chars)).length <= maxChars; const top = Math.min(lines[first].length, MAX_LINE_CHARS); let fits = 0; let over = top; @@ -145,7 +167,7 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu // The search ends with `fits + 1` measured as too long, or as `top`. const found = fits; for (let chars = found + 2; chars <= found + 4 && chars < top; chars++) if (fitsAt(chars)) fits = chars; - return { success: true, data: pageWith(fits) }; + return pageAnswer(pageWith(fits)); } const splitsPair = (text: string, at: number): boolean => @@ -155,8 +177,8 @@ const splitsPair = (text: string, at: number): boolean => /** * Raw text from `offsetChars`, at most `limitChars` code units (one more to - * keep a surrogate pair whole), stopped sooner where its data would pass - * `maxChars` as JSON, if there is one. + * keep a surrogate pair whole), stopped sooner where its page's text would + * pass `maxChars`, if there is one. */ export function characterPage(path: string, file: FileText, offsetChars: number, limitChars: number, maxChars: number | undefined, extraNotes: string[] = []): ToolResult { const text = file.raw; @@ -172,7 +194,7 @@ export function characterPage(path: string, file: FileText, offsetChars: number, // asked to: at a cut line's end, for the rest a line page's note names. const stoppedNote = (to: number, left: number) => `Stopped at character ${to} to stay within the ${maxChars}-character result limit; read the rest of what you asked for with offsetChars=${to} and limitChars=${left}.`; - const page = (to: number, note: string | undefined) => ({ + const page = (to: number, note: string | undefined): Page => ({ path, totalChars: text.length, offsetChars: start, @@ -181,23 +203,18 @@ export function characterPage(path: string, file: FileText, offsetChars: number, content: text.slice(start, to), }); const whole = page(end, joined(notes)); - if (maxChars === undefined || JSON.stringify(whole).length <= maxChars) return { success: true, data: whole }; + if (maxChars === undefined || pageText(whole).length <= maxChars) return pageAnswer(whole); // As many characters as fit beside the rest of the page, which takes the - // most with the largest offset it could stop at and the most left. - const overhead = JSON.stringify({ ...page(end, joined([stoppedNote(end, end - start), ...notes])), nextOffsetChars: end, content: '' }).length; - // A page moves by whole characters, a surrogate pair at once, so it never - // stops inside one. + // most with the largest offset it could stop at and the most left. A page + // moves by whole characters, a surrogate pair at once, so it never stops + // inside one. + const overhead = pageText({ ...page(end, joined([stoppedNote(end, end - start), ...notes])), nextOffsetChars: end, content: '' }).length + sectionLead('content'); const step = (at: number) => (splitsPair(text, at + 1) ? 2 : 1); - let to = start; - for (let used = overhead; to < end;) { - const cost = jsonCharLength(text, to) + (step(to) === 2 ? jsonCharLength(text, to + 1) : 0); - if (used + cost > maxChars) break; - used += cost; - to += step(to); - } + let to = Math.min(end, start + Math.max(0, maxChars - overhead)); + if (splitsPair(text, to)) to--; // A page always moves on, by one character at least, though a limit too // small for the page's own fields then passes it. Where that reaches the // page's own end, it holds all it was asked for, so it names no stop. if (to === start) to = Math.min(end, start + step(start)); - return { success: true, data: page(to, joined([to < end ? stoppedNote(to, end - to) : null, ...notes])) }; + return pageAnswer(page(to, joined([to < end ? stoppedNote(to, end - to) : null, ...notes]))); } diff --git a/src/modules/workspace/result-text.ts b/src/modules/workspace/result-text.ts new file mode 100644 index 000000000..71a1248d5 --- /dev/null +++ b/src/modules/workspace/result-text.ts @@ -0,0 +1,104 @@ +import type { ToolResult } from '../../types/events.js'; + +/** + * The text a model reads of read's, grep's, edit's and edit_many's answers: + * a header of the answer's own fields, one `key: value` to a line in the + * fields' order, then its sections, each a `--- label ---` line and the lines + * under it, an empty one left out. The answer's fields themselves reach a + * script as its structuredContent (ToolResult.structured), and a page or a + * receipt is held to the call's result limit as this text measures it. + */ + +const LINE_BREAK = /[\n\r\u2028\u2029]/; + +/** + * A value as a header line, a section's label or a listed line writes it: as + * it is, or, where it holds a line break, as a JSON string. A path can hold + * one, and so can a note naming it; written as it is, it would start a line + * that reads as one of the answer's own, and the quoted form gives back + * exactly what the value holds. JSON leaves U+2028 and U+2029 as they are, + * so they're escaped too. + */ +export function shownValue(value: string | number | boolean | null): string { + if (typeof value !== 'string') return String(value); + return LINE_BREAK.test(value) ? JSON.stringify(value).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029') : value; +} + +/** A header line: a field's key and its value as shownValue writes it. */ +export const fieldLine = (key: string, value: string | number | boolean | null): string => `${key}: ${shownValue(value)}`; + +/** A section's text, its label and then its lines; an empty section is no text. */ +export const sectionText = (label: string, body: string): string => (body === '' ? '' : `--- ${label} ---\n${body}`); + +/** The answer's text: its header lines and then its sections, each on lines of its own. */ +export const answerText = (header: string[], sections: string[] = []): string => + [...header, ...sections.filter((section) => section !== '')].join('\n'); + +/** How much a section with any lines adds to its answer's text before them: the line feed before its label, and the label's line. */ +export const sectionLead = (label: string): number => `\n--- ${label} ---\n`.length; + +/** A success whose text is what the model reads and whose fields reach a script as its structuredContent. */ +export const textAnswer = (text: string, structured: Record): ToolResult => + ({ success: true, data: [{ type: 'text', text }], structured }); + +/** A grep result: the matches of one version of a file. */ +export interface GrepResult { + file: string; + state?: string; + version: string; + lineEndings?: string; + matches: Array<{ line: number; text: string; context?: string[] }>; +} + +/** What grep found, as its structuredContent holds it. */ +export interface GrepAnswer { + pattern: string; + results: GrepResult[]; + totalMatches: number; + note?: string; + skipped?: Array<{ file: string; reason: string; size?: number; mimeType?: string }>; + incomplete?: Array<{ path: string; reason: string }>; +} + +/** + * grep's text: its pattern, match count and note, then a section for each + * version of a file it matched, labelled with the file, the version, the + * file's state and its line ends where it has them, then what it skipped and + * where its search was incomplete. A match's line reads `N:text`, as grep + * writes one, and where the call asked for context, each line around it reads + * `N-text`, with `--` between one match's lines and the next's. Each match's + * context starts `contextBefore` lines before it, or at the file's first line. + */ +export function grepText(answer: GrepAnswer, contextBefore: number): string { + const header = [fieldLine('pattern', answer.pattern), fieldLine('totalMatches', answer.totalMatches)]; + if (answer.note !== undefined) header.push(fieldLine('note', answer.note)); + const sections = answer.results.map((result) => { + const label = `${shownValue(result.file)}: ${result.version}` + + (result.state !== undefined ? ` (${result.state})` : '') + + (result.lineEndings !== undefined ? `; lineEndings: ${result.lineEndings}` : ''); + const lines: string[] = []; + for (const match of result.matches) { + if (match.context === undefined) { + lines.push(`${String(match.line).padStart(6)}:${match.text}`); + continue; + } + if (lines.length > 0) lines.push('--'); + const first = Math.max(1, match.line - contextBefore); + match.context.forEach((text, k) => { + const n = first + k; + lines.push(`${String(n).padStart(6)}${n === match.line ? ':' : '-'}${text}`); + }); + } + return sectionText(label, lines.join('\n')); + }); + if (answer.skipped !== undefined) { + sections.push(sectionText('skipped', answer.skipped.map((skip) => { + const facts = [...(skip.size !== undefined ? [`${skip.size} bytes`] : []), ...(skip.mimeType !== undefined ? [skip.mimeType] : [])]; + return `${shownValue(skip.file)}: ${shownValue(skip.reason)}${facts.length ? ` (${facts.join(', ')})` : ''}`; + }).join('\n'))); + } + if (answer.incomplete !== undefined) { + sections.push(sectionText('incomplete', answer.incomplete.map((gap) => `${shownValue(gap.path)}: ${shownValue(gap.reason)}`).join('\n'))); + } + return answerText(header, sections); +} diff --git a/src/modules/workspace/text.ts b/src/modules/workspace/text.ts index 0a00189d9..68fcd0d8f 100644 --- a/src/modules/workspace/text.ts +++ b/src/modules/workspace/text.ts @@ -1,10 +1,9 @@ import { isUtf8 } from 'node:buffer'; /** - * A stored file's text as read shows it and grep searches it, lines numbered - * the way `cat -n` numbers them, with a long one cut, and a string's length - * as JSON, which a budget for a stored result counts. Read, grep and edit's - * answers use them. + * A stored file's text as read shows it and grep searches it, and lines + * numbered the way `cat -n` numbers them, with a long one cut. Read, grep and + * edit's answers use them. */ /** Whether the bytes open with a UTF-8 byte order mark. */ @@ -93,30 +92,3 @@ export function numberedLine(line: string, n: number, maxChars = MAX_LINE_CHARS, export function numbered(lines: string[], start: number, maxChars = MAX_LINE_CHARS): string { return lines.map((line, i) => numberedLine(line, start + i, maxChars)).join('\n'); } - -/** - * How many characters a string takes inside JSON's quotes, as JSON.stringify - * writes it: a quote, a backslash or a control character is escaped, and so - * is a lone surrogate. - */ -export function jsonLength(value: string): number { - let length = 0; - for (let i = 0; i < value.length; i++) length += jsonCharLength(value, i); - return length; -} - -/** The JSON length of the code unit at `i`, as jsonLength counts it. */ -export function jsonCharLength(value: string, i: number): number { - const c = value.charCodeAt(i); - if (c === 0x22 || c === 0x5c) return 2; - if (c < 0x20) return c === 0x08 || c === 0x09 || c === 0x0a || c === 0x0c || c === 0x0d ? 2 : 6; - if (c >= 0xd800 && c <= 0xdbff) { - const next = value.charCodeAt(i + 1); - return next >= 0xdc00 && next <= 0xdfff ? 1 : 6; - } - if (c >= 0xdc00 && c <= 0xdfff) { - const before = i > 0 ? value.charCodeAt(i - 1) : 0; - return before >= 0xd800 && before <= 0xdbff ? 1 : 6; - } - return 1; -} diff --git a/test/tool-result-cap-fit.test.ts b/test/tool-result-cap-fit.test.ts index 9d894283f..9077cce38 100644 --- a/test/tool-result-cap-fit.test.ts +++ b/test/tool-result-cap-fit.test.ts @@ -141,9 +141,9 @@ const countWrites = (writes: string[]) => (framework: AgentFramework) => { * characters, beside the page's metadata), and is sized to that cap, not * to the `share` of the turn that cut the result. */ const assertPageForLaterRead = (limit: number, quotedPath: string, share: number) => { - const overhead = quotedPath.length + 256; - assert.ok(limit * 6 + overhead <= 4000, `a page of ${limit} comes back within the cap`); - assert.ok(limit * 6 + overhead > share, `a page of ${limit} is sized to the cap, not to the share of ${share}`); + const overhead = (JSON.parse(quotedPath) as string).length + 256; + assert.ok(limit + overhead <= 4000, `a page of ${limit} comes back within the cap`); + assert.ok(limit + overhead > share, `a page of ${limit} is sized to the cap, not to the share of ${share}`); }; /** agent_settings get's view of an agent's inline cap. */ diff --git a/test/tool-result-spill.test.ts b/test/tool-result-spill.test.ts index d6de1459f..fdddd52f0 100644 --- a/test/tool-result-spill.test.ts +++ b/test/tool-result-spill.test.ts @@ -344,7 +344,7 @@ describe('tool-result spill completion (issue #89)', () => { id: `page-${pages}`, name: 'workspace--read', input, callerAgentName: 'prime', }); assert.strictEqual(result.success, true, result.error); - const page = result.data as { content: string; offsetChars: number; nextOffsetChars: number | null }; + const page = result.structured as { content: string; offsetChars: number; nextOffsetChars: number | null }; assert.strictEqual(page.offsetChars, recovered.length); assert.ok(page.content.length > 0); // Exercise the actual stored/live spill policy on each read result. @@ -358,7 +358,8 @@ describe('tool-result spill completion (issue #89)', () => { const outcome = storage.spilled.get(`page-${pages}`)!; assert.strictEqual(outcome.filePath, null, 'read-back must not spill again'); assert.ok(outcome.text.length <= cap); - assert.strictEqual(JSON.parse(outcome.text).content, page.content); + assert.strictEqual(outcome.text, (result.data as Array<{ text: string }>)[0]!.text, 'the page is stored as its text'); + assert.ok(outcome.text.endsWith(`\n--- content ---\n${page.content}`), 'which ends with its content'); recovered += page.content; pages++; if (page.nextOffsetChars === null) break; @@ -395,7 +396,7 @@ describe('tool-result spill completion (issue #89)', () => { id: 'tail', name: 'read', input: { path: ref[1], offsetChars: expected.length - 100, limitChars: 100 }, }); assert.strictEqual(lastPage.success, true); - assert.deepStrictEqual((lastPage.data as { content: string }).content, expected.toString().slice(-100)); + assert.deepStrictEqual((lastPage.structured as { content: string }).content, expected.toString().slice(-100)); } finally { await h.framework.stop(); rmSync(h.tempDir, { recursive: true, force: true }); @@ -455,7 +456,7 @@ describe('tool-result spill completion (issue #89)', () => { id: 'grep', name: 'grep', input: { pattern: 'repo#2999 ', path: refMatch[1] }, }); assert.strictEqual(grep.success, true, grep.error); - const matches = (grep.data as { results: Array<{ matches: Array<{ line: number; text: string }> }> }) + const matches = (grep.structured as { results: Array<{ matches: Array<{ line: number; text: string }> }> }) .results.flatMap((r) => r.matches); assert.deepStrictEqual(matches.map((m) => [m.line, m.text]), [[3000, body.split('\n')[2999]]]); } finally { diff --git a/test/workspace-direct-write.test.ts b/test/workspace-direct-write.test.ts index e4fc2ec92..0f94202a6 100644 --- a/test/workspace-direct-write.test.ts +++ b/test/workspace-direct-write.test.ts @@ -91,7 +91,7 @@ function call(m: WorkspaceModule, name: string, input: Record = async function data>(m: WorkspaceModule, name: string, input: Record = {}): Promise { const result = await call(m, name, input); assert.equal(result.success, true, `${name} ${JSON.stringify(input)}: ${result.error}`); - return result.data as T; + return (result.structured ?? result.data) as T; } async function failure(m: WorkspaceModule, name: string, input: Record): Promise { diff --git a/test/workspace-direct.test.ts b/test/workspace-direct.test.ts index 43c37705b..410c28c73 100644 --- a/test/workspace-direct.test.ts +++ b/test/workspace-direct.test.ts @@ -162,7 +162,7 @@ function call(m: WorkspaceModule, name: string, input: Record = async function data>(m: WorkspaceModule, name: string, input: Record = {}): Promise { const result = await call(m, name, input); assert.equal(result.success, true, `${name} ${JSON.stringify(input)}: ${result.error}`); - return result.data as T; + return (result.structured ?? result.data) as T; } async function failure(m: WorkspaceModule, name: string, input: Record): Promise { @@ -398,7 +398,7 @@ test("a direct mount's read loads none of a file over the size limit and grep on const read = await countingLoads(() => call(m, 'read', { path: 'ref/huge.txt' })); assert.deepEqual([read.result.error, read.loaded], [over('ref/huge.txt'), 0]); const grep = await countingLoads(() => call(m, 'grep', { pattern: 'y', path: 'ref/huge.txt' })); - assert.deepEqual((grep.result.data as { skipped: unknown[] }).skipped, [{ file: 'ref/huge.txt', reason: 'over the size limit (10 bytes)', size: 100_000 }]); + assert.deepEqual((grep.result.structured as { skipped: unknown[] }).skipped, [{ file: 'ref/huge.txt', reason: 'over the size limit (10 bytes)', size: 100_000 }]); assert.equal(grep.loaded, 8192); // Grown past it after its open's fstat: no more than the limit and a byte. const grows = join(env.ref, 'grows.txt'); @@ -409,7 +409,7 @@ test("a direct mount's read loads none of a file over the size limit and grep on writeFileSync(grows, 'x'.repeat(8)); const grownGrep = await changedAfterFstat(grows, () => appendFileSync(grows, 'x'.repeat(1000)), () => countingLoads(() => call(m, 'grep', { pattern: 'x', path: 'ref/grows.txt' }))); - assert.deepEqual((grownGrep.result.data as { skipped: unknown[] }).skipped, [{ file: 'ref/grows.txt', reason: 'over the size limit (10 bytes)', size: 1008 }]); + assert.deepEqual((grownGrep.result.structured as { skipped: unknown[] }).skipped, [{ file: 'ref/grows.txt', reason: 'over the size limit (10 bytes)', size: 1008 }]); assert.ok(grownGrep.loaded <= 11, `loaded ${grownGrep.loaded}`); }); @@ -419,7 +419,7 @@ test("a direct mount's read and grep say so where reading a file fails", async ( const m = await env.open(); const failing = { read: () => async function () { throw Object.assign(new Error('EIO: i/o error, read'), { code: 'EIO' }); } }; assert.equal((await withFileHandle(failing, () => call(m, 'read', { path: 'ref/a.txt' }))).error, 'Unable to read file: ref/a.txt (EIO)'); - assert.deepEqual((await withFileHandle(failing, () => call(m, 'grep', { pattern: 'needle' }))).data, { + assert.deepEqual((await withFileHandle(failing, () => call(m, 'grep', { pattern: 'needle' }))).structured, { pattern: 'needle', results: [], totalMatches: 0, @@ -666,7 +666,7 @@ test("a direct mount takes each spelling of a path as a 'chronicle' mount does", */ function alike(result: ToolResult): unknown { if (!result.success) return { refused: true }; - const d = result.data as Record; + const d = (result.structured ?? result.data) as Record; if (Array.isArray(d)) return { text: d[0].text.split('\n')[0] }; return { path: d.path, @@ -1204,7 +1204,7 @@ test("a mount configured beneath a moved direct root is nested beneath a directo }); const fromStart = await answers(started); assert.match(fromStart.read.error ?? '', /lies in mount "work", nested in mount "ref"/); - assert.equal((fromStart.own.data as { content: string }).content, ' 1\tb own'); + assert.equal((fromStart.own.structured as { content: string }).content, ' 1\tb own'); assert.deepEqual(await answers(moved), fromStart); }); @@ -1229,7 +1229,7 @@ test("a direct root moved to a directory beneath its own configured path nests n nested: await call(m, 'read', { path: 'ref/sub/work/x.txt' }), }); const fromStart = await answers(started); - assert.equal((fromStart.own.data as { content: string }).content, ' 1\tsub own'); + assert.equal((fromStart.own.structured as { content: string }).content, ' 1\tsub own'); assert.match(fromStart.nested.error ?? '', /lies in mount "work", nested in mount "ref"/); assert.deepEqual(await answers(moved), fromStart); }); @@ -1282,7 +1282,7 @@ test("a mount configured at a moved direct root's own path is nested nowhere in disk: await m.readFileFromDisk('ref/sub/f.txt').then(() => 'read', (err: unknown) => (err instanceof WorkspaceReadError ? err.code : String(err))), }); const fromStart = await answers(started); - assert.equal((fromStart.read.data as { content: string }).content, ' 1\tsub f'); + assert.equal((fromStart.read.structured as { content: string }).content, ' 1\tsub f'); assert.equal(fromStart.disk, 'read'); assert.deepEqual(await answers(moved), fromStart); }); @@ -2397,7 +2397,10 @@ test("a direct mount's read shows a file's text as a 'chronicle' mount's does: l const out: Array<[Record, number | undefined, Record]> = []; for (const [input, cap] of reads) { const r = await m.handleToolCall({ id: 'read', name: 'read', input, ...(cap !== undefined ? { resultMaxChars: cap } : {}) }); - out.push([input, cap, r.success ? r.data as Record : { error: r.error }]); + // Every page a call with a result limit asked for fits it, as its text measures it. + const text = r.success ? (r.data as Array<{ text: string }>)[0]!.text : ''; + if (cap !== undefined) assert.ok(text.length <= cap, `${String(input.path)}${'offsetChars' in input ? ' in characters' : ''} at ${cap}: ${text.length}`); + out.push([input, cap, r.success ? r.structured as Record : { error: r.error }]); } return out; }; @@ -2412,11 +2415,10 @@ test("a direct mount's read shows a file's text as a 'chronicle' mount's does: l assert.match(page('bad.txt', 24000).note, /isn't valid UTF-8/); assert.equal(page('mixed.txt', 24000).content, ' 1\ta\n 2\tb\n 3\tc\r\r\n 4\td'); assert.equal(page('long.txt', 24000).content, ` 1\t${'x'.repeat(2000)} [… read the rest with offsetChars=2000 and limitChars=58000]\n 2\ty`); - // Every page a call with a result limit asked for is a page, and fits it. + // Every page a call asked for is a page. for (const [input, cap, d] of fromDisk) { const label = `${String(input.path)}${'offsetChars' in input ? ' in characters' : ''} at ${cap}`; assert.equal(typeof d.content, 'string', `${label}: ${JSON.stringify(d).slice(0, 200)}`); - if (cap !== undefined) assert.ok(JSON.stringify(d).length <= cap, `${label}: ${JSON.stringify(d).length}`); } assert.ok(page('big.txt', 24000).toLine < 3000); assert.equal(page('long.txt', undefined).content.length, ' 1\t'.length + 60000 + '\n 2\ty'.length); diff --git a/test/workspace-edits.test.ts b/test/workspace-edits.test.ts index 5e198a893..02147203c 100644 --- a/test/workspace-edits.test.ts +++ b/test/workspace-edits.test.ts @@ -13,6 +13,11 @@ import { JsStore } from '@animalabs/chronicle'; import { WorkspaceModule } from '../src/modules/workspace/index.js'; import { decodeForEdit, planFile, type FilePlan } from '../src/modules/workspace/edit-plan.js'; import { editReceipt, fitted, shownRegions } from '../src/modules/workspace/edit-text.js'; + +/** A receipt's text, which the model reads and the cap measures. */ +const receiptText = (receipt: ToolResult): string => (receipt.data as Array<{ text: string }>)[0]!.text; +/** The changed regions a receipt shows, from its fields. */ +const changedOf = (receipt: ToolResult): string => (receipt.structured as { changed: string }).changed; import type { ToolResult } from '../src/types/events.js'; function setup(t: TestContext, opts: { autoMaterialize?: boolean; maxFileSize?: number } = {}) { @@ -44,7 +49,7 @@ function setup(t: TestContext, opts: { autoMaterialize?: boolean; maxFileSize?: const stored = async (rel: string): Promise => { const result = await call('read', { path: `work/${rel}`, offsetChars: 0, limitChars: 1_000_000 }); assert.equal(result.success, true, result.error); - return (result.data as { content: string }).content; + return (result.structured as { content: string }).content; }; /** A file the workspace holds, as a listing takes it in. */ const file = async (rel: string, content: string | Buffer): Promise => { @@ -60,9 +65,10 @@ const errorOf = (result: ToolResult): string => { return result.error ?? ''; }; +/** A success's fields: a receipt's, a page's or grep's, which a script reads as structuredContent, or another tool's data. */ const dataOf = (result: ToolResult): Record => { assert.equal(result.success, true, result.error); - return result.data as Record; + return (result.structured ?? result.data) as Record; }; /** Half of a surrogate pair without the other half. */ @@ -315,7 +321,7 @@ test("a receipt reads the stored bytes of its lines that end in a CR in one walk return Reflect.apply(search, this, args) as number; }, }); - const changed = editReceipt('edit', [{ path: 'work/c.txt', plan: last.plan, notes: [] }], 1, 10_000_000).changed as string; + const changed = changedOf(editReceipt('edit', [{ path: 'work/c.txt', plan: last.plan, notes: [] }], 1, 10_000_000)); assert.equal((changed.match(/\ty\r\r/g) ?? []).length, 300); assert.ok(finds >= 300 && finds < 3_000, `${finds} searches of the bytes`); }); @@ -1287,7 +1293,7 @@ test("a conflict says what a sync does from what disk holds now, whatever kind w const copy = "Sync the path to keep disk's copy and give up the workspace's, or materialize it with force to write the workspace's over disk's."; const one = await call('edit', { path: 'work/gone.txt', oldString: 'agent draft', newString: 'agent edit' }); assert.equal(one.success, true, one.error); - assert.equal((one.data as { note?: string }).note, + assert.equal((one.structured as { note?: string }).note, `The path is in conflict with disk (deleted-on-disk): the edit is in the workspace's copy. ${deletion}`); // A refusal over the same file says the same. assert.ok(errorOf(await call('edit', { path: 'work/gone.txt', oldString: 'absent', newString: 'x' })).endsWith(deletion)); @@ -1299,7 +1305,7 @@ test("a conflict says what a sync does from what disk holds now, whatever kind w ], }); assert.equal(many.success, true, many.error); - assert.deepEqual((many.data as { files: Array<{ note?: string }> }).files.map((file) => file.note), [ + assert.deepEqual((many.structured as { files: Array<{ note?: string }> }).files.map((file) => file.note), [ `work/changed-then-gone.txt is in conflict with disk (both-changed): edit 1 is in the workspace's copy. ${deletion}`, `work/gone-then-back.txt is in conflict with disk (deleted-on-disk): edit 2 is in the workspace's copy. ${copy}`, `work/turned-binary.txt is in conflict with disk (both-changed): edit 3 is in the workspace's copy. ${copy}`, @@ -1311,7 +1317,7 @@ test("a conflict says what a sync does from what disk holds now, whatever kind w } assert.equal(errorOf(await call('read', { path: 'work/gone.txt' })), 'File not found: work/gone.txt'); assert.equal(disk('gone-then-back.txt')!.toString(), 'another writer\n'); - assert.match(JSON.stringify((await call('read', { path: 'work/gone-then-back.txt' })).data), /another writer/); + assert.match(JSON.stringify((await call('read', { path: 'work/gone-then-back.txt' })).structured), /another writer/); assert.equal(errorOf(await call('read', { path: 'work/turned-binary.txt' })), 'work/turned-binary.txt is on disk but not stored in the workspace (binary or over the size limit).'); assert.deepEqual([...disk('turned-binary.txt')!], binary); @@ -1420,10 +1426,9 @@ test("a receipt keeps to the call's resultMaxChars, naming the regions past it b const edits = Array.from({ length: 30 }, (_, i) => ({ path: 'work/g.txt', oldString: `row ${i * 10} `, newString: `ROW ${i * 10} ` })); const max = 3000; const result = await call('edit_many', { edits }, max); - const receipt = dataOf(result); - assert.ok(JSON.stringify(receipt).length <= max, `${JSON.stringify(receipt).length} characters, within ${max}`); - assert.match(receipt.changed as string, /\[not shown, to keep this answer within 3,000 characters: lines \d+–\d+, /); - assert.match(receipt.changed as string, /^work\/g\.txt\n 1\tROW 0 /); + assert.ok(receiptText(result).length <= max, `${receiptText(result).length} characters, within ${max}`); + assert.match(changedOf(result), /\[not shown, to keep this answer within 3,000 characters: lines \d+–\d+, /); + assert.match(changedOf(result), /^work\/g\.txt\n 1\tROW 0 /); }); test("edit's refusal and receipt keep to its call's resultMaxChars too", async (t) => { @@ -1436,20 +1441,22 @@ test("edit's refusal and receipt keep to its call's resultMaxChars too", async ( assert.match(refusal, /^The edit did nothing: oldString occurs 60 times in work\/l\.txt /); assert.ok(errorOf(await call('edit', { path: 'work/l.txt', oldString: 'line', newString: 'LINE' })).length > 1_000); // A change of eighty lines is named by its lines. - const receipt = dataOf(await call('edit', { path: 'work/l.txt', oldString: 'line 0 ', newString: `${`${'A'.repeat(30)}\n`.repeat(80)}line 0 ` }, 1_000)); - assert.ok(JSON.stringify(receipt).length <= 1_000, `${JSON.stringify(receipt).length} characters, within 1,000`); - assert.match(receipt.changed as string, /^\[not shown, to keep this answer within 1,000 characters: lines 1–\d+\]$/); + const receipt = await call('edit', { path: 'work/l.txt', oldString: 'line 0 ', newString: `${`${'A'.repeat(30)}\n`.repeat(80)}line 0 ` }, 1_000); + assert.ok(receiptText(receipt).length <= 1_000, `${receiptText(receipt).length} characters, within 1,000`); + assert.match(changedOf(receipt), /^\[not shown, to keep this answer within 1,000 characters: lines 1–\d+\]$/); }); -test("a receipt's regions are measured as its JSON holds them, where a quote or a backslash takes two characters", async (t) => { +test("a receipt's regions are measured as its text holds them, where a quote or a backslash takes one character", async (t) => { const { call, file } = setup(t); await file('q.txt', 'start\n'); - // Eight lines of sixty characters, 120 each as JSON: counted as typed they - // fit what this receipt has left for its regions, and as JSON they don't. + // Eight lines of sixty characters, 120 each as JSON: counted as the text + // holds them they fit what this receipt has left for its regions, and as + // JSON they wouldn't. const quoted = Array.from({ length: 8 }, () => '"\\'.repeat(30)).join('\n'); - const receipt = dataOf(await call('edit', { path: 'work/q.txt', oldString: 'start', newString: quoted }, 1_000)); - assert.ok(JSON.stringify(receipt).length <= 1_000, `${JSON.stringify(receipt).length} characters, within 1,000`); - assert.equal(receipt.changed, '[not shown, to keep this answer within 1,000 characters: lines 1–8]'); + const receipt = await call('edit', { path: 'work/q.txt', oldString: 'start', newString: quoted }, 1_000); + assert.ok(receiptText(receipt).length <= 1_000, `${receiptText(receipt).length} characters, within 1,000`); + assert.ok(JSON.stringify(receipt.structured).length > 1_000, 'as JSON the receipt would pass 1,000'); + assert.equal(changedOf(receipt), quoted.split('\n').map((line, i) => `${String(i + 1).padStart(6)}\t${line}`).join('\n')); }); test('a receipt names a region too long to show by its lines without building it whole', async () => { @@ -1501,7 +1508,7 @@ test('a receipt keeps to its cap with the longest notes it adds after its budget const plan = markedPlan(lines, Array.from({ length: 25 }, (_, k) => lines - 400 + k * 15)); const files = [0, 1].map((i) => ({ path: `work/f${i}.txt`, plan, notes: i ? [`Disk's copy of work/f${i}.txt had changed.`] : [] })); for (let maxChars = 700; maxChars <= 1_700; maxChars++) { - const length = JSON.stringify(editReceipt('edit_many', files, 50, maxChars)).length; + const length = receiptText(editReceipt('edit_many', files, 50, maxChars)).length; if (length > maxChars) assert.fail(`${length} characters at a cap of ${maxChars}`); } }); @@ -1519,9 +1526,9 @@ test("a receipt keeps aside the longest note any of its files could add, not its let named = 0; for (let maxChars = 700; maxChars <= 1_700; maxChars++) { const receipt = editReceipt('edit_many', files, 26, maxChars); - const length = JSON.stringify(receipt).length; + const length = receiptText(receipt).length; if (length > maxChars) assert.fail(`${length} characters at a cap of ${maxChars}`); - if (twenty.test(receipt.changed as string)) named++; + if (twenty.test(changedOf(receipt))) named++; } assert.ok(named > 0, "no receipt named twenty of the first file's regions"); }); @@ -1543,9 +1550,9 @@ test("edit_many's receipt keeps to its cap with a last line that counts each of let counted = 0; for (let maxChars = 700; maxChars <= 2_500; maxChars++) { const receipt = editReceipt('edit_many', files, 125, maxChars); - const length = JSON.stringify(receipt).length; + const length = receiptText(receipt).length; if (length > maxChars) assert.fail(`${length} characters at a cap of ${maxChars}`); - if (allFour.test(receipt.changed as string)) counted++; + if (allFour.test(changedOf(receipt))) counted++; } assert.ok(counted > 0, 'no receipt counted all four'); }); @@ -1560,10 +1567,10 @@ test('a receipt keeps aside only what its own call can add: one edit of a file o assert.ok('plan' in planned); const files = [{ path: 'work/long.txt', plan: planned.plan, notes: [] }]; for (const [tool, start] of [['edit', /^ {5}1\ty/], ['edit_many', /^work\/long\.txt\n {5}1\ty/]] as const) { - const whole = JSON.stringify(editReceipt(tool, files, 1, 1_000_000)).length; + const whole = receiptText(editReceipt(tool, files, 1, 1_000_000)).length; const receipt = editReceipt(tool, files, 1, whole + 300); - assert.match(receipt.changed as string, start, tool); - assert.ok(JSON.stringify(receipt).length <= whole + 300); + assert.match(changedOf(receipt), start, tool); + assert.ok(receiptText(receipt).length <= whole + 300); } }); @@ -1578,9 +1585,9 @@ test("what a receipt keeps aside doesn't grow with a line's length: twenty edits const planned = planFile(decodeForEdit(bytes)!, bytes, Array.from({ length: 20 }, (_, k) => ({ index: k, oldText: `m${k}|`, newText: `M${k}|`, expected: 1 }))); assert.ok('plan' in planned); const files = [{ path: 'work/line.txt', plan: planned.plan, notes: [] }]; - const whole = JSON.stringify(editReceipt('edit_many', files, 20, 1_000_000)).length; + const whole = receiptText(editReceipt('edit_many', files, 20, 1_000_000)).length; // Whether the region shows with `spare` characters past the whole receipt. - return (spare: number) => /^work\/line\.txt\n {5}1\t/.test(editReceipt('edit_many', files, 20, whole + spare).changed as string); + return (spare: number) => /^work\/line\.txt\n {5}1\t/.test(changedOf(editReceipt('edit_many', files, 20, whole + spare))); }; const short = showsAt(start); const long = showsAt(start + 'a'.repeat(1_000_000)); @@ -1611,19 +1618,21 @@ test('a receipt lists its files while they fit and counts the rest, and counts t const edits = (from: string, to: string) => Array.from({ length: 30 }, (_, i) => ({ path: `work/f${i}.txt`, oldString: `${from} ${i}`, newString: `${to} ${i}` })); - const short = dataOf(await call('edit_many', { edits: edits('value', 'VALUE') }, 1500)); - assert.ok(JSON.stringify(short).length <= 1500, `${JSON.stringify(short).length} characters`); + const shortAnswer = await call('edit_many', { edits: edits('value', 'VALUE') }, 1500); + const short = dataOf(shortAnswer); + assert.ok(receiptText(shortAnswer).length <= 1500, `${receiptText(shortAnswer).length} characters`); const listed = (short.files as unknown[]).length; assert.ok(listed > 0 && listed < 30); assert.equal(short.filesNotListed, 30 - listed); // Its last line counts them too. assert.match(short.changed as string, new RegExp(`\\[To keep this answer within 1,500 characters, it leaves out ${30 - listed} of the call's 30 files[^\\]]*\\.\\]$`)); - const longer = dataOf(await call('edit_many', { edits: edits('VALUE', 'value') }, 2800)); - assert.ok(JSON.stringify(longer).length <= 2800, `${JSON.stringify(longer).length} characters`); + const longerAnswer = await call('edit_many', { edits: edits('VALUE', 'value') }, 2200); + const longer = dataOf(longerAnswer); + assert.ok(receiptText(longerAnswer).length <= 2200, `${receiptText(longerAnswer).length} characters`); assert.equal((longer.files as unknown[]).length, 30); assert.match(longer.changed as string, /^work\/f0\.txt\n 1\tvalue 0\n\n/); - assert.match(longer.changed as string, /\[To keep this answer within 2,800 characters, it leaves out the changed regions of \d+ more files\.\]$/); + assert.match(longer.changed as string, /\[To keep this answer within 2,200 characters, it leaves out the changed regions of \d+ more files\.\]$/); }); test('at the least cap a residence or an agent can set, a refusal still names its problems, however long their labels or paths', async (t) => { diff --git a/test/workspace-file-tools.test.ts b/test/workspace-file-tools.test.ts index 93590313d..fc49c9e76 100644 --- a/test/workspace-file-tools.test.ts +++ b/test/workspace-file-tools.test.ts @@ -30,7 +30,7 @@ function setup(t: TestContext, opts: { autoMaterialize?: boolean; maxFileSize?: const stored = async (rel: string): Promise => { const result = await call('read', { path: `work/${rel}`, offsetChars: 0, limitChars: 100_000 }); assert.equal(result.success, true, result.error); - return (result.data as { content: string }).content; + return (result.structured as { content: string }).content; }; return { workspace, store, dir, call, disk, stored }; } @@ -500,7 +500,7 @@ test('a write, edit or delete whose store throws after recording it says the cha // the workspace made, not a file disk may have deleted. const edited = await call('edit', { path: 'work/w.txt', oldString: 'new', newString: 'newer' }); assert.equal(edited.success, true, edited.error); - assert.doesNotMatch(String((edited.data as { note?: string }).note ?? ''), /nothing records/); + assert.doesNotMatch(String((edited.structured as { note?: string }).note ?? ''), /nothing records/); }); test('on a mount that pushes, a write, edit or delete whose store throws after recording it says its push didn\'t run, and a materialize takes it to disk', async (t) => { @@ -593,14 +593,14 @@ test('on a mount that pushes, a write, edit or delete whose store throws after r test('a final line end ends the last line, and an empty file reads as empty', async (t) => { const { dir, call } = setup(t); writeFileSync(join(dir, 'nl.txt'), 'a\nb\n'); - assert.deepEqual((await call('read', { path: 'work/nl.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/nl.txt' })).structured, { path: 'work/nl.txt', totalLines: 2, fromLine: 1, toLine: 2, content: ' 1\ta\n 2\tb', }); writeFileSync(join(dir, 'blank.txt'), 'a\n\n'); - assert.equal(((await call('read', { path: 'work/blank.txt' })).data as { totalLines: number }).totalLines, 2, + assert.equal(((await call('read', { path: 'work/blank.txt' })).structured as { totalLines: number }).totalLines, 2, 'an empty line before the final line end is a line'); writeFileSync(join(dir, 'empty.txt'), ''); - assert.deepEqual((await call('read', { path: 'work/empty.txt' })).data, { path: 'work/empty.txt', totalLines: 0, content: '' }); + assert.deepEqual((await call('read', { path: 'work/empty.txt' })).structured, { path: 'work/empty.txt', totalLines: 0, content: '' }); }); test('grep numbers lines as read does: no match on an empty line after the last', async (t) => { @@ -608,7 +608,7 @@ test('grep numbers lines as read does: no match on an empty line after the last' writeFileSync(join(dir, 'g.txt'), 'x\n'); const result = await call('grep', { pattern: '^$' }); assert.equal(result.success, true, result.error); - assert.deepEqual((result.data as { results: unknown[] }).results, []); + assert.deepEqual((result.structured as { results: unknown[] }).results, []); }); test('grep matches and shows a CRLF file\'s lines without the CR, as read shows them', async (t) => { @@ -620,7 +620,7 @@ test('grep matches and shows a CRLF file\'s lines without the CR, as read shows const result = await call('grep', { pattern: '^foo$', contextBefore: 1, contextAfter: 1 }); assert.equal(result.success, true, result.error); type Result = { file: string; lineEndings?: string; matches: unknown[] }; - const byFile = Object.fromEntries((result.data as { results: Result[] }).results.map((r) => [r.file, r])); + const byFile = Object.fromEntries((result.structured as { results: Result[] }).results.map((r) => [r.file, r])); assert.deepEqual(byFile['work/crlf.txt'], { file: 'work/crlf.txt', state: 'synced', version: 'workspace', lineEndings: 'CRLF', matches: [{ line: 2, text: 'foo', context: ['alpha', 'foo', 'omega'] }], @@ -636,7 +636,7 @@ test('grep matches and shows a CRLF file\'s lines without the CR, as read shows const note = noteFor('1 file searched has CRLF line ends: work/crlf.txt (CRLF)'); for (const pattern of ['\\r', '\r', 'o[\\r]?$', '\\\\\\r', '\\x0d', '\\x0D', '\\u000d', '\\u000D', '\\cM', '\\cm', '\\015', '[\\x0d]', '\\\r']) { const cr = await call('grep', { pattern, path: 'work/crlf.txt' }); - const data = cr.data as { results: Array<{ matches: unknown[] }>; note?: string }; + const data = cr.structured as { results: Array<{ matches: unknown[] }>; note?: string }; assert.equal(data.note, note, JSON.stringify(pattern)); if (pattern !== 'o[\\r]?$') assert.deepEqual(data.results, [], JSON.stringify(pattern)); } @@ -644,11 +644,11 @@ test('grep matches and shows a CRLF file\'s lines without the CR, as read shows // end, get no note. for (const [pattern, path] of [['foo', 'work/crlf.txt'], ['\\\\r', 'work/crlf.txt'], ['\\\\x0d', 'work/crlf.txt'], ['\\\\cM', 'work/crlf.txt'], ['\\\\u000d', 'work/crlf.txt'], ['\\R', 'work/crlf.txt'], ['\\X0D', 'work/crlf.txt'], ['\\U000D', 'work/crlf.txt'], ['\\CM', 'work/crlf.txt'], ['\\r', 'work/lf.txt']]) { - const data = (await call('grep', { pattern, path })).data as object; + const data = (await call('grep', { pattern, path })).structured as object; assert.equal('note' in data, false, JSON.stringify([pattern, path])); } writeFileSync(join(dir, 'lone.txt'), 'a\rb\n'); - const lone = (await call('grep', { pattern: '\\r', path: 'work/lone.txt' })).data as { results: Array<{ matches: unknown[] }> }; + const lone = (await call('grep', { pattern: '\\r', path: 'work/lone.txt' })).structured as { results: Array<{ matches: unknown[] }> }; assert.equal('note' in lone, false); assert.deepEqual(lone.results[0].matches, [{ line: 1, text: 'a\rb' }]); // Any CRLF file searched gives the note, whatever comes after it: here an LF @@ -659,9 +659,9 @@ test('grep matches and shows a CRLF file\'s lines without the CR, as read shows writeFileSync(join(dir, 'many', 'a.txt'), 'x\n'); writeFileSync(join(dir, 'many', 'b.txt'), 'x\r\n'); writeFileSync(join(dir, 'many', 'c.txt'), 'x\n'); - assert.equal(((await call('grep', { pattern: '\\r', path: 'work/many' })).data as { note?: string }).note, + assert.equal(((await call('grep', { pattern: '\\r', path: 'work/many' })).structured as { note?: string }).note, noteFor('1 file searched has CRLF line ends: work/many/b.txt (CRLF)')); - const globbed = (await call('grep', { pattern: 'x|\\r', path: 'work/many', glob: 'many/a.txt' })).data as { results: Array<{ file: string }> }; + const globbed = (await call('grep', { pattern: 'x|\\r', path: 'work/many', glob: 'many/a.txt' })).structured as { results: Array<{ file: string }> }; assert.deepEqual(globbed.results.map((r) => r.file), ['work/many/a.txt']); assert.equal('note' in globbed, false); }); @@ -674,36 +674,36 @@ test("grep says a BOM isn't part of the first line when a pattern for U+FEFF sea + "and a BOM isn't part of the first line. read with offsetChars shows any BOM the workspace's copy has."; const note = noteFor('work/bom.txt'); for (const pattern of ['\\uFEFF', '^\\ufeff', '\uFEFF', '\\\uFEFF']) { - const data = (await call('grep', { pattern, path: 'work/bom.txt' })).data as { results: unknown[]; note?: string }; + const data = (await call('grep', { pattern, path: 'work/bom.txt' })).structured as { results: unknown[]; note?: string }; assert.equal(data.note, note, JSON.stringify(pattern)); assert.deepEqual(data.results, [], JSON.stringify(pattern)); } // An escaped backslash names no U+FEFF, nor does \U, an identity escape, // and a U+FEFF inside a line, in a file with no BOM, is found with no note. for (const pattern of ['\\\\uFEFF', '\\UFEFF']) { - assert.equal('note' in ((await call('grep', { pattern, path: 'work/bom.txt' })).data as object), false, JSON.stringify(pattern)); + assert.equal('note' in ((await call('grep', { pattern, path: 'work/bom.txt' })).structured as object), false, JSON.stringify(pattern)); } - const inner = (await call('grep', { pattern: '\\uFEFF', path: 'work/inner.txt' })).data as { results: Array<{ matches: unknown[] }> }; + const inner = (await call('grep', { pattern: '\\uFEFF', path: 'work/inner.txt' })).structured as { results: Array<{ matches: unknown[] }> }; assert.equal('note' in inner, false); assert.deepEqual(inner.results[0].matches, [{ line: 1, text: 'a\uFEFFb' }]); // A CR pattern over the BOM file, which has no CRLF end, and a U+FEFF // pattern over a CRLF file with no BOM, get no note. writeFileSync(join(dir, 'crlf.txt'), 'x\r\n'); - assert.equal('note' in ((await call('grep', { pattern: '\\r', path: 'work/bom.txt' })).data as object), false); - assert.equal('note' in ((await call('grep', { pattern: '\\uFEFF', path: 'work/crlf.txt' })).data as object), false); + assert.equal('note' in ((await call('grep', { pattern: '\\r', path: 'work/bom.txt' })).structured as object), false); + assert.equal('note' in ((await call('grep', { pattern: '\\uFEFF', path: 'work/crlf.txt' })).structured as object), false); // Any BOM file searched gives the note, whatever comes after it: a file // without one is searched on either side of the BOM file. mkdirSync(join(dir, 'boms')); writeFileSync(join(dir, 'boms', 'a.txt'), 'x\n'); writeFileSync(join(dir, 'boms', 'b.txt'), Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('x\n')])); writeFileSync(join(dir, 'boms', 'c.txt'), 'x\n'); - assert.equal(((await call('grep', { pattern: '\\uFEFF', path: 'work/boms' })).data as { note?: string }).note, noteFor('work/boms/b.txt')); + assert.equal(((await call('grep', { pattern: '\\uFEFF', path: 'work/boms' })).structured as { note?: string }).note, noteFor('work/boms/b.txt')); // A pattern that names both, over a file with a BOM and CRLF ends, gets both // notes, the CR's first. writeFileSync(join(dir, 'both.txt'), Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('x\r\n')])); const crNote = "The pattern names a CR, and 1 file searched has CRLF line ends: work/both.txt (CRLF). Lines are matched as read shows them, " + "which leaves out a CRLF end's CR unless the line's own text ends in a CR. read with offsetChars shows every CR in the workspace's copy."; - const both = (await call('grep', { pattern: '^\\uFEFF|\\r$', path: 'work/both.txt' })).data as { results: unknown[]; note?: string }; + const both = (await call('grep', { pattern: '^\\uFEFF|\\r$', path: 'work/both.txt' })).structured as { results: unknown[]; note?: string }; assert.equal(both.note, `${crNote} ${noteFor('work/both.txt')}`); assert.deepEqual(both.results, []); }); @@ -725,22 +725,22 @@ test("grep's note on a pattern for a CR or U+FEFF names the files searched that + "read with offsetChars shows every CR in the workspace's copy."; // On main, \r$ found these files' lines; here no line keeps a CRLF end's // CR, so the note names the files, mixed ends as such, sorted as results are. - const cr = (await call('grep', { pattern: '\\r$' })).data as { results: unknown[]; totalMatches: number; note?: string; skipped?: Array<{ file: string }> }; + const cr = (await call('grep', { pattern: '\\r$' })).structured as { results: unknown[]; totalMatches: number; note?: string; skipped?: Array<{ file: string }> }; assert.deepEqual([cr.results, cr.totalMatches], [[], 0]); assert.deepEqual(cr.skipped?.map((s) => s.file), ['work/big.txt', 'work/blob.bin']); assert.equal(cr.note, 'The pattern names a CR, and 3 files searched have CRLF line ends: ' + `work/both.json (CRLF), work/mixed.cfg (mixed), work/win.bat (CRLF). ${tail}`); - const boms = (await call('grep', { pattern: '^\\uFEFF' })).data as { results: unknown[]; note?: string }; + const boms = (await call('grep', { pattern: '^\\uFEFF' })).structured as { results: unknown[]; note?: string }; assert.deepEqual(boms.results, []); assert.equal(boms.note, 'The pattern names U+FEFF, and 2 files searched start with a BOM: work/bom.txt, work/both.json. ' + "Lines are matched as read shows them, and a BOM isn't part of the first line. read with offsetChars shows any BOM the workspace's copy has."); // Past 20 files the count stays exact, and the rest are found by narrowing. mkdirSync(join(dir, 'many')); for (let i = 0; i < 23; i++) writeFileSync(join(dir, 'many', `f${String(i).padStart(2, '0')}.txt`), 'x\r\n'); - const many = (await call('grep', { pattern: '\\r', path: 'work/many' })).data as { note?: string }; + const many = (await call('grep', { pattern: '\\r', path: 'work/many' })).structured as { note?: string }; const first20 = Array.from({ length: 20 }, (_, i) => `work/many/f${String(i).padStart(2, '0')}.txt (CRLF)`).join(', '); assert.equal(many.note, `The pattern names a CR, and 23 files searched have CRLF line ends: ${first20}, and 3 more; narrow with path or glob. ${tail}`); - const narrowed = (await call('grep', { pattern: '\\r', path: 'work/many', glob: 'many/f2*.txt' })).data as { note?: string }; + const narrowed = (await call('grep', { pattern: '\\r', path: 'work/many', glob: 'many/f2*.txt' })).structured as { note?: string }; assert.equal(narrowed.note, 'The pattern names a CR, and 3 files searched have CRLF line ends: ' + `work/many/f20.txt (CRLF), work/many/f21.txt (CRLF), work/many/f22.txt (CRLF). ${tail}`); }); @@ -750,7 +750,7 @@ test('text copied from a read or a grep of a CRLF file matches in an edit, and t const bom = Buffer.from([0xef, 0xbb, 0xbf]); writeFileSync(join(dir, 'w.txt'), Buffer.concat([bom, Buffer.from('one\r\ntwo\r\nthree\r\n')])); // Two lines as the read shows them, each without its number and tab. - const page = (await call('read', { path: 'work/w.txt' })).data as { content: string }; + const page = (await call('read', { path: 'work/w.txt' })).structured as { content: string }; const copied = page.content.split('\n').slice(0, 2).map((line) => line.slice(line.indexOf('\t') + 1)).join('\n'); assert.equal(copied, 'one\ntwo'); const edit = await call('edit', { path: 'work/w.txt', oldString: copied, newString: 'ONE\nTWO' }); @@ -758,7 +758,7 @@ test('text copied from a read or a grep of a CRLF file matches in an edit, and t assert.deepEqual(disk('w.txt'), Buffer.concat([bom, Buffer.from('ONE\r\nTWO\r\nthree\r\n')])); // A grep match's text copies the same way. const found = await call('grep', { pattern: '^three$', path: 'work/w.txt' }); - const text = (found.data as { results: Array<{ matches: Array<{ text: string }> }> }).results[0].matches[0].text; + const text = (found.structured as { results: Array<{ matches: Array<{ text: string }> }> }).results[0].matches[0].text; const second = await call('edit', { path: 'work/w.txt', oldString: text, newString: 'THREE' }); assert.equal(second.success, true, second.error); assert.deepEqual(disk('w.txt'), Buffer.concat([bom, Buffer.from('ONE\r\nTWO\r\nTHREE\r\n')])); @@ -769,7 +769,7 @@ test('lines whose own text ends in a CR, copied from a read, edit the lines they // Lines 1 and 2 end in a CR before their CRLF end; lines 3 and 4 hold the // same text with a CRLF end alone. writeFileSync(join(dir, 'cr.txt'), 'x\r\r\ny\r\r\nx\r\ny\r\n'); - const page = (await call('read', { path: 'work/cr.txt' })).data as { content: string; note: string }; + const page = (await call('read', { path: 'work/cr.txt' })).structured as { content: string; note: string }; assert.equal(page.note, 'Line ends are CRLF, shown without the CR, except on 2 lines whose own text ends in a CR, which show both CRs. ' + 'An edit matches text as read shows it, and keeps the CRLF ends.'); const copied = page.content.split('\n').slice(0, 2).map((line) => line.slice(line.indexOf('\t') + 1)).join('\n'); @@ -780,7 +780,7 @@ test('lines whose own text ends in a CR, copied from a read, edit the lines they // A grep match's text copies the same way. writeFileSync(join(dir, 'one.txt'), 'w\r\r\nw\r\n'); const found = await call('grep', { pattern: '^w', path: 'work/one.txt' }); - const texts = (found.data as { results: Array<{ matches: Array<{ text: string }> }> }).results[0].matches.map((m) => m.text); + const texts = (found.structured as { results: Array<{ matches: Array<{ text: string }> }> }).results[0].matches.map((m) => m.text); assert.deepEqual(texts, ['w\r\r', 'w']); const second = await call('edit', { path: 'work/one.txt', oldString: texts[0], newString: 'V' }); assert.equal(second.success, true, second.error); @@ -800,11 +800,11 @@ test('an offset past the last line answers empty, with the line count and no ran const { dir, call } = setup(t); writeFileSync(join(dir, 'nl.txt'), 'a\nb\n'); writeFileSync(join(dir, 'empty.txt'), ''); - assert.deepEqual((await call('read', { path: 'work/empty.txt', offset: 2 })).data, { + assert.deepEqual((await call('read', { path: 'work/empty.txt', offset: 2 })).structured, { path: 'work/empty.txt', totalLines: 0, note: 'Line 2 is past the end: the file has 0 lines.', content: '', }); for (const offset of [3, 10]) { - assert.deepEqual((await call('read', { path: 'work/nl.txt', offset })).data, { + assert.deepEqual((await call('read', { path: 'work/nl.txt', offset })).structured, { path: 'work/nl.txt', totalLines: 2, note: `Line ${offset} is past the end: the file has 2 lines.`, content: '', }); } @@ -822,7 +822,7 @@ test("an edit applies to disk's text when disk changed and the store didn't, and const edit = await call('edit', { path: 'work/s.txt', oldString: 'line', newString: 'LINE' }); assert.equal(edit.success, true, edit.error); - assert.match((edit.data as { note: string }).note, /^Disk's copy had changed since the workspace last agreed with it, so it was taken in first, and the edit applied to that text\.$/); + assert.match((edit.structured as { note: string }).note, /^Disk's copy had changed since the workspace last agreed with it, so it was taken in first, and the edit applied to that text\.$/); assert.equal(disk('s.txt')!.toString(), 'v2 LINE from another writer\n'); assert.equal(await stored('s.txt'), 'v2 LINE from another writer\n'); @@ -855,7 +855,7 @@ test('an edit of a draft whose disk copy also changed applies to the draft and n + "The path is in conflict with disk (both-changed): the edit looked in the workspace's copy. Sync the path to keep disk's copy and give up the workspace's, or materialize it with force to write the workspace's over disk's."); const edit = await call('edit', { path: 'work/k.txt', oldString: 'draft', newString: 'draft, edited' }); assert.equal(edit.success, true, edit.error); - assert.equal((edit.data as { note: string }).note, + assert.equal((edit.structured as { note: string }).note, "The path is in conflict with disk (both-changed): the edit is in the workspace's copy. Sync the path to keep disk's copy and give up the workspace's, or materialize it with force to write the workspace's over disk's."); assert.equal(await stored('k.txt'), 'agent draft, edited\n'); assert.equal(disk('k.txt')!.toString(), 'another writer\n'); @@ -878,16 +878,16 @@ test("a read shows disk's text when disk changed and the store didn't, and says assert.equal((await call('read', { path: 'work/r.txt' })).success, true); writeFileSync(join(dir, 'r.txt'), 'v2 from another writer\n'); const taken = "Disk's copy had changed since the workspace last agreed with it, so it was taken in first, and this read shows that text."; - assert.deepEqual((await call('read', { path: 'work/r.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/r.txt' })).structured, { path: 'work/r.txt', totalLines: 1, fromLine: 1, toLine: 1, note: taken, content: ' 1\tv2 from another writer', }); // That's the workspace's copy now, so the next read has nothing to say. - assert.deepEqual((await call('read', { path: 'work/r.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/r.txt' })).structured, { path: 'work/r.txt', totalLines: 1, fromLine: 1, toLine: 1, content: ' 1\tv2 from another writer', }); // A character page settles and says so too. writeFileSync(join(dir, 'r.txt'), 'v3\n'); - assert.deepEqual((await call('read', { path: 'work/r.txt', offsetChars: 0 })).data, { + assert.deepEqual((await call('read', { path: 'work/r.txt', offsetChars: 0 })).structured, { path: 'work/r.txt', totalChars: 3, offsetChars: 0, nextOffsetChars: null, note: taken, content: 'v3\n', }); }); @@ -907,7 +907,7 @@ test('a read of a draft whose disk copy also changed shows the draft and names t assert.equal((await call('read', { path: 'work/k.txt' })).success, true); assert.equal((await call('write', { path: 'work/k.txt', content: 'agent draft\n' })).success, true); writeFileSync(join(dir, 'k.txt'), 'another writer\n'); - assert.deepEqual((await call('read', { path: 'work/k.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/k.txt' })).structured, { path: 'work/k.txt', totalLines: 1, fromLine: 1, toLine: 1, note: "The path is in conflict with disk (both-changed): this read shows the workspace's copy. " + "Sync the path to keep disk's copy and give up the workspace's, or materialize it with force to write the workspace's over disk's.", @@ -922,7 +922,7 @@ test("a read of a draft whose disk copy turned binary says a sync keeps disk's c assert.equal((await call('read', { path: 'work/m.txt' })).success, true); assert.equal((await call('write', { path: 'work/m.txt', content: 'agent draft\n' })).success, true); writeFileSync(join(dir, 'm.txt'), Buffer.from([0, 1, 2, 0, 255, 0])); - assert.equal(((await call('read', { path: 'work/m.txt' })).data as { note: string }).note, + assert.equal(((await call('read', { path: 'work/m.txt' })).structured as { note: string }).note, "The path is in conflict with disk (both-changed): this read shows the workspace's copy. " + "Sync the path to keep disk's copy and give up the workspace's, or materialize it with force to write the workspace's over disk's."); // The workspace can't hold disk's copy, so the sync takes nothing in: the @@ -940,7 +940,7 @@ test('a read of a draft whose disk copy was deleted says a sync takes the deleti assert.equal((await call('read', { path: `work/${name}` })).success, true); assert.equal((await call('write', { path: `work/${name}`, content: 'agent draft\n' })).success, true); rmSync(join(dir, name)); - assert.deepEqual((await call('read', { path: `work/${name}` })).data, { + assert.deepEqual((await call('read', { path: `work/${name}` })).structured, { path: `work/${name}`, totalLines: 1, fromLine: 1, toLine: 1, note: "The path is in conflict with disk (deleted-on-disk): this read shows the workspace's copy. " + "Sync the path to take disk's deletion, which removes the workspace's copy, or materialize it with force to write the workspace's copy back to disk.", @@ -953,7 +953,7 @@ test('a read of a draft whose disk copy was deleted says a sync takes the deleti assert.equal((await call('read', { path: 'work/f.txt' })).success, true); assert.equal((await call('write', { path: 'work/f.txt', content: 'agent draft\n' })).success, true); writeFileSync(join(dir, 'f.txt'), 'another writer\n'); - const noteOf = async (path: string) => ((await call('read', { path })).data as { note: string }).note; + const noteOf = async (path: string) => ((await call('read', { path })).structured as { note: string }).note; assert.match(await noteOf('work/f.txt'), /^The path is in conflict with disk \(both-changed\): .*Sync the path to keep disk's copy and give up the workspace's/); rmSync(join(dir, 'f.txt')); assert.equal(await noteOf('work/f.txt'), @@ -973,7 +973,7 @@ test("a read whose disk copy can't be checked says it shows the workspace's copy assert.equal((await call('read', { path: 'work/u.txt' })).success, true); rmSync(join(dir, 'u.txt')); mkdirSync(join(dir, 'u.txt')); - assert.deepEqual((await call('read', { path: 'work/u.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/u.txt' })).structured, { path: 'work/u.txt', totalLines: 1, fromLine: 1, toLine: 1, note: "The path couldn't be checked against disk (a directory), so this read shows the workspace's copy.", content: ' 1\theld', @@ -988,7 +988,7 @@ test("a read of a held file whose mount's root is gone says disk couldn't be che writeFileSync(join(dir, 'u.txt'), 'held\n'); assert.equal((await call('read', { path: 'work/u.txt' })).success, true); renameSync(dir, join(dir, '..', 'gone')); - assert.deepEqual((await call('read', { path: 'work/u.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/u.txt' })).structured, { path: 'work/u.txt', totalLines: 1, fromLine: 1, toLine: 1, note: "The path couldn't be checked against disk (the mount root is unavailable), so this read shows the workspace's copy.", content: ' 1\theld', @@ -1000,7 +1000,7 @@ test("a read of a file disk lacks, with nothing to say whether disk deleted it, // An entry from before the evidence (as three-way's legacyEntry makes it). const bytes = Buffer.from('legacy line\n'); store.treeSet('workspace/work/tree', 'l.txt', { blobHash: store.storeBlob(bytes, 'text/plain'), size: bytes.byteLength, mode: 0o644 }); - assert.deepEqual((await call('read', { path: 'work/l.txt' })).data, { + assert.deepEqual((await call('read', { path: 'work/l.txt' })).structured, { path: 'work/l.txt', totalLines: 1, fromLine: 1, toLine: 1, note: "Disk has no copy of this file, and nothing records whether it was deleted there, so this read shows the workspace's copy.", content: ' 1\tlegacy line', @@ -1106,7 +1106,7 @@ test("an edit whose disk copy can't be checked says it used the workspace's copy + `move or remove that directory on disk and materialize again, or write the file under another path. ${used}`); } else { assert.equal(edit.success, true, edit.error); - assert.equal((edit.data as { note: string }).note, used); + assert.equal((edit.structured as { note: string }).note, used); } assert.equal(await stored('u.txt'), 'held, edited\n'); assert.equal(errorOf(await call('edit', { path: 'work/u.txt', oldString: 'zzz', newString: 'x' })), @@ -1158,7 +1158,7 @@ test('an edit of a draft whose disk copy also changed, on a mount that pushes, n // push meets it, so its answer names none. const agreed = await call('edit', { path: 'work/k.txt', oldString: 'agent draft, edited', newString: 'another writer' }); assert.equal(agreed.success, true, agreed.error); - assert.equal((agreed.data as { note?: string }).note, undefined, 'no conflict remains to name'); + assert.equal((agreed.structured as { note?: string }).note, undefined, 'no conflict remains to name'); assert.equal(await stored('k.txt'), 'another writer\n'); assert.equal(disk('k.txt')!.toString(), 'another writer\n'); const after = await call('materialize', { path: 'work/k.txt' }); @@ -1381,7 +1381,7 @@ for (const autoMaterialize of [false, true]) { store.treeSet('workspace/work/tree', 'l.txt', { blobHash: store.storeBlob(bytes, 'text/plain'), size: bytes.byteLength, mode: 0o644 }); const edit = await call('edit', { path: 'work/l.txt', oldString: 'legacy', newString: 'edited' }); assert.equal(edit.success, true, edit.error); - assert.equal((edit.data as { note?: string }).note, + assert.equal((edit.structured as { note?: string }).note, "Disk has no copy of this file, and nothing records whether it was deleted there, so the edit applied to the workspace's copy. " + (autoMaterialize ? 'Its push put the file back on disk' : 'A materialize of the path puts it back on disk') + '; if it was deleted there on purpose, delete it here too.'); @@ -1429,7 +1429,7 @@ test("an edit of a draft whose disk copy was deleted says what a sync and a forc + `The path is in conflict with disk (deleted-on-disk): the edit looked in the workspace's copy. ${DELETION_WAYS}`); const edit = await call('edit', { path: 'work/f.txt', oldString: 'draft', newString: 'draft, edited' }); assert.equal(edit.success, true, edit.error); - assert.equal((edit.data as { note?: string }).note, + assert.equal((edit.structured as { note?: string }).note, `The path is in conflict with disk (deleted-on-disk): the edit is in the workspace's copy. ${DELETION_WAYS}`); // Each way out does what the note says. assert.equal((await call('sync', { path: 'work/s.txt' })).success, true); diff --git a/test/workspace-glob-mount-prefix.test.ts b/test/workspace-glob-mount-prefix.test.ts index 80e4c8a5e..7f2e0e757 100644 --- a/test/workspace-glob-mount-prefix.test.ts +++ b/test/workspace-glob-mount-prefix.test.ts @@ -53,7 +53,7 @@ function setup(t: TestContext, extra: Record = {}) { const grepFiles = async (input: Record) => { const r = await workspace.handleToolCall({ id: 'r', name: 'grep', input }); assert.equal(r.success, true, JSON.stringify(r)); - return ((r.data as { results: Array<{ file: string }> }).results).map((x) => x.file).sort(); + return ((r.structured as { results: Array<{ file: string }> }).results).map((x) => x.file).sort(); }; const refusal = async (name: 'glob' | 'grep', input: Record) => { const r = await workspace.handleToolCall({ id: 'x', name, input }); diff --git a/test/workspace-glob-unclosed-brace.test.ts b/test/workspace-glob-unclosed-brace.test.ts index 296f01436..0de1052dc 100644 --- a/test/workspace-glob-unclosed-brace.test.ts +++ b/test/workspace-glob-unclosed-brace.test.ts @@ -18,7 +18,7 @@ import { join } from 'node:path'; const WORKSPACE = new URL('../src/modules/workspace/index.js', import.meta.url).href; const HANG_GUARD_MS = 60_000; -interface Answer { success: boolean; isError?: boolean; error?: string; data?: unknown } +interface Answer { success: boolean; isError?: boolean; error?: string; data?: unknown; structured?: unknown } /** One workspace tool call, made in a child process against a mount `project` * holding `src/a.ts`, `src/c.ts` and `lib/b.ts`. Returns its result, or null @@ -118,7 +118,7 @@ test('groups that close still match, each by its own braces (control)', () => { assert.ok(grep, `grep with the glob "{src,lib}/{a,b}.ts" didn't answer within ${HANG_GUARD_MS / 1000} s`); assert.equal(grep.success, true, JSON.stringify(grep)); assert.deepEqual( - (grep.data as { results: Array<{ file: string }> }).results.map((r) => r.file).sort(), + (grep.structured as { results: Array<{ file: string }> }).results.map((r) => r.file).sort(), ['project/lib/b.ts', 'project/src/a.ts'], ); }); diff --git a/test/workspace-grep-filter.test.ts b/test/workspace-grep-filter.test.ts index 7f59ec4a3..6f1c2d985 100644 --- a/test/workspace-grep-filter.test.ts +++ b/test/workspace-grep-filter.test.ts @@ -56,7 +56,7 @@ async function opened(t: TestContext, mount: Partial, more: string[ async function grepped(m: WorkspaceModule, input: Record): Promise { const res = await m.handleToolCall({ id: 't', name: 'grep', input: { pattern: 'needle', ...input } }); assert.equal(res.success, true, `grep ${JSON.stringify(input)} failed: ${res.error}`); - return ((res.data as { results: Array<{ file: string }> }).results).map((r) => r.file).sort(); + return ((res.structured as { results: Array<{ file: string }> }).results).map((r) => r.file).sort(); } /** The files a glob finds, sorted. */ @@ -77,7 +77,7 @@ async function globRefusal(m: WorkspaceModule, input: Record): /** The error a grep for the needle is refused with. */ async function refusal(m: WorkspaceModule, input: Record): Promise { const res = await m.handleToolCall({ id: 't', name: 'grep', input: { pattern: 'needle', ...input } }); - assert.equal(res.success, false, `grep ${JSON.stringify(input)} unexpectedly succeeded: ${JSON.stringify(res.data)}`); + assert.equal(res.success, false, `grep ${JSON.stringify(input)} unexpectedly succeeded: ${JSON.stringify(res.structured)}`); assert.equal(res.isError, true, 'a refusal is an error the agent sees'); return String(res.error); } @@ -204,7 +204,7 @@ for (const [kind, mount] of KINDS) { ] as const) { const res = await m.handleToolCall({ id: 't', name, input }); assert.equal(res.success, true, `${name} ${JSON.stringify(input)} failed: ${res.error}`); - const incomplete = (res.data as { incomplete?: Array<{ path: string; reason: string }> }).incomplete ?? []; + const incomplete = ((res.structured ?? res.data) as { incomplete?: Array<{ path: string; reason: string }> }).incomplete ?? []; assert.deepEqual(incomplete.map((region) => region.path), ['work/a/b'], name); assert.match(incomplete[0]!.reason, /it lies in mount "deep", nested in this one/, name); } diff --git a/test/workspace-include.test.ts b/test/workspace-include.test.ts index 6fece8c89..25bd775b9 100644 --- a/test/workspace-include.test.ts +++ b/test/workspace-include.test.ts @@ -81,7 +81,7 @@ function madeFifo(path: string): boolean { async function call(m: WorkspaceModule, name: string, input: Record): Promise { const res = await m.handleToolCall({ id: 't', name, input }); assert.equal(res.success, true, `${name} ${JSON.stringify(input)} failed: ${res.error}`); - return res.data; + return res.structured ?? res.data; } async function refused(m: WorkspaceModule, name: string, input: Record): Promise { @@ -381,7 +381,7 @@ describe('mount include', () => { const top = ((await call(m, 'ls', { path: 'work' })).entries as Array<{ name: string; type: string }>).map((e) => `${e.type} ${e.name}`); assert.deepEqual(top, ['directory x', 'file gate.json'], `${onDisk}: the listing names x`); const res = await m.handleToolCall({ id: 't', name: tool, input: input as Record }); - said.push(`${tool} ${res.success ? JSON.stringify(res.data) : `refused: ${res.error}`}`); + said.push(`${tool} ${res.success ? JSON.stringify(res.structured ?? res.data) : `refused: ${res.error}`}`); if (onDisk === 'a file') assert.equal(readFileSync(join(env.dir, 'x'), 'utf8'), 'a file outside the include\n', `${tool} leaves the file`); } return said; @@ -1444,7 +1444,7 @@ describe('mount include', () => { for (const path of ['work/notes.md', 'work/sub/../notes.md']) { const read = await m.handleToolCall({ id: 't', name: 'read', input: { path }, callerAgentName: 'resident' }); assert.equal(read.success, true, `${path}: ${read.error}`); - assert.match(JSON.stringify(read.data), /generated/, path); + assert.match(JSON.stringify(read.structured), /generated/, path); const write = await m.handleToolCall({ id: 't', name: 'write', input: { path, content: 'x' }, callerAgentName: 'resident' }); assert.match(String(write.error), /Generated file is read-only/, path); } @@ -1606,7 +1606,7 @@ describe('mount include', () => { for (const [path, use] of [['work/a/b/x.txt', 'deep/x.txt'], ['work/a/b/gate.json', 'deep/gate.json']]) { for (const [name, extra] of tools) { const res = await m.handleToolCall({ id: 't', name, input: { path, ...extra } }); - const said = JSON.stringify(res.success ? res.data : String(res.error)); + const said = JSON.stringify(res.success ? res.structured ?? res.data : String(res.error)); assert.ok(said.includes(`use \\"${use}\\"`), `${name} ${path}: ${said}`); assert.ok(!said.includes(OUTSIDE), `${name} ${path}: ${said}`); } diff --git a/test/workspace-materialize-freshness.test.ts b/test/workspace-materialize-freshness.test.ts index 6f6256f03..00af919ca 100644 --- a/test/workspace-materialize-freshness.test.ts +++ b/test/workspace-materialize-freshness.test.ts @@ -137,7 +137,7 @@ test('syncing the path resolves the divergence in the disk direction; a full syn assert.ok(((still.data as MaterializeData).skipped ?? []).some((s) => s.reason.includes('script.sh')), 'a full sync leaves the conflict for an explicit resolution'); const read = await call(module, 'read', { path: 'work/script.sh' }); - assert.match(String((read.data as { content: string }).content), /echo v2/, 'the workspace edit survives'); + assert.match(String((read.structured as { content: string }).content), /echo v2/, 'the workspace edit survives'); // The remedy the skip reason points at: sync this path to adopt the disk version. await call(module, 'sync', { path: 'work/script.sh' }); diff --git a/test/workspace-nested-mount-roots.test.ts b/test/workspace-nested-mount-roots.test.ts index bdefcba3c..9418c98e7 100644 --- a/test/workspace-nested-mount-roots.test.ts +++ b/test/workspace-nested-mount-roots.test.ts @@ -68,8 +68,8 @@ function mount(name: string, path: string, extra: Partial = {}): Mo return { name, path, mode: 'read-write', watch: 'never', autoMaterialize: false, ...extra }; } -async function call(m: WorkspaceModule, name: string, input: Record): Promise<{ success: boolean; error?: string; data?: any }> { - return m.handleToolCall({ id: 't', name, input }) as Promise<{ success: boolean; error?: string; data?: any }>; +async function call(m: WorkspaceModule, name: string, input: Record): Promise<{ success: boolean; error?: string; data?: any; structured?: any }> { + return m.handleToolCall({ id: 't', name, input }) as Promise<{ success: boolean; error?: string; data?: any; structured?: any }>; } /** The refusal a file tool gives a path that reaches a nested mount's directory through a link the mount follows. */ @@ -81,7 +81,7 @@ function leadsInto(path: string, outer: string, inner: string, use: string): str async function readsBack(m: WorkspaceModule, path: string, text: string): Promise { const read = await call(m, 'read', { path }); assert.equal(read.success, true, `${path}: ${read.error}`); - assert.match(read.data.content, new RegExp(text), path); + assert.match(read.structured.content, new RegExp(text), path); } /** The skipped reason for a path in a materialize's or a sync's result. */ @@ -341,7 +341,7 @@ describe('a link into a nested mount\'s directory, on a mount that follows symli await readsBack(m, 'inner/x.txt', 'inner file'); const top = await call(m, 'read', { path: 'work/l/top.txt' }); assert.equal(top.success, true, top.error); - assert.match(top.data.content, /outer file under a/); + assert.match(top.structured.content, /outer file under a/); for (const [path, use] of [['work/l/b/new.txt', 'inner/new.txt'], ['work/l/b/deep/new.txt', 'inner/deep/new.txt']] as const) { const write = await call(m, 'write', { path, content: 'via work\n' }); assert.equal(write.error, leadsInto(path, 'work', 'inner', use)); @@ -420,8 +420,8 @@ describe('a link into a nested mount\'s directory, on a mount that follows symli assert.equal(read.error, leadsInto('outer/flink.txt', 'outer', 'inner', 'inner/x.txt')); for (const path of ['outer', 'outer/flink.txt']) { const grep = await call(m, 'grep', { pattern: 'copy', path }); - assert.deepEqual(grep.data.results, [], `grep of ${path} searched none of the copy held from before`); - assert.deepEqual(grep.data.skipped, [{ file: 'outer/flink.txt', reason: leadsInto('outer/flink.txt', 'outer', 'inner', 'inner/x.txt') }], path); + assert.deepEqual(grep.structured.results, [], `grep of ${path} searched none of the copy held from before`); + assert.deepEqual(grep.structured.skipped, [{ file: 'outer/flink.txt', reason: leadsInto('outer/flink.txt', 'outer', 'inner', 'inner/x.txt') }], path); } // A write made now, with the link in place, is refused at the tool // (anima-research/agent-framework#530's third case). @@ -460,8 +460,8 @@ describe('a link into a nested mount\'s directory, on a mount that follows symli }, () => call(m, 'grep', { pattern: 'main|OTHER', path: 'work' })); assert.ok(switched, 'a branch was selected while grep asked'); const settled = await call(m, 'grep', { pattern: 'main|OTHER', path: 'work' }); - assert.notEqual(settled.data.results[0]?.state, 'synced', "z.txt's state on the other branch isn't main's"); - assert.deepEqual(grep.data.results, settled.data.results, "grep gave the other branch's text with that branch's states"); + assert.notEqual(settled.structured.results[0]?.state, 'synced', "z.txt's state on the other branch isn't main's"); + assert.deepEqual(grep.structured.results, settled.structured.results, "grep gave the other branch's text with that branch's states"); }); for (const autoMaterialize of [false, true]) test(`a draft made before a file link into it took the path's place gets the path to use from a push, unforced and forced (autoMaterialize ${autoMaterialize})`, async (t) => { diff --git a/test/workspace-parent-link.test.ts b/test/workspace-parent-link.test.ts index 19e4e0b78..f0aabfd4a 100644 --- a/test/workspace-parent-link.test.ts +++ b/test/workspace-parent-link.test.ts @@ -116,7 +116,7 @@ class Env { async function call(m: WorkspaceModule, name: string, input: Record): Promise { const res = await m.handleToolCall({ id: 't', name, input }); assert.equal(res.success, true, `${name} ${JSON.stringify(input)} failed: ${res.error}`); - return res.data; + return res.structured ?? res.data; } /** Run `fn` with node:fs/promises' `name` replaced by `make(original)`, as the code under test sees it. */ diff --git a/test/workspace-path-spelling.test.ts b/test/workspace-path-spelling.test.ts index 7f33affa3..a51fde8de 100644 --- a/test/workspace-path-spelling.test.ts +++ b/test/workspace-path-spelling.test.ts @@ -108,7 +108,7 @@ test('a search whose path is spelled in another case names what it finds as disk const glob = await call('glob', { pattern: '*.ts', path: 'work/SRC' }); assert.deepEqual((glob.data as { matches: Array<{ path: string }> }).matches.map((m) => m.path), ['work/src/a.ts']); const grep = await call('grep', { pattern: 'const', path: 'work/Src' }); - assert.deepEqual((grep.data as { results: Array<{ file: string }> }).results.map((r) => r.file), ['work/src/a.ts']); + assert.deepEqual((grep.structured as { results: Array<{ file: string }> }).results.map((r) => r.file), ['work/src/a.ts']); const ls = await call('ls', { path: 'work/SRC' }); assert.equal((ls.data as { path: string }).path, 'work/src'); assert.deepEqual(pushed(), [], 'nothing was taken in under the spelling given'); @@ -125,13 +125,13 @@ test('a read, write, edit and delete spelled in another case act on disk\'s file pushed(); const read = await call('read', { path: 'work/SRC/A.TS', offsetChars: 0, limitChars: 100 }); - assert.deepEqual(read.data, { + assert.deepEqual(read.structured, { path: 'work/src/a.ts', totalChars: 20, offsetChars: 0, nextOffsetChars: null, content: 'export const a = 1;\n', }); const write = await call('write', { path: 'work/SRC/b.ts', content: 'export const b = 2;\n' }); assert.equal((write.data as { path: string }).path, 'work/src/b.ts'); const edit = await call('edit', { path: 'work/Src/B.ts', oldString: '2', newString: '3' }); - assert.equal((edit.data as { path: string }).path, 'work/src/b.ts'); + assert.equal((edit.structured as { path: string }).path, 'work/src/b.ts'); assert.equal(readFileSync(join(dir, 'src', 'b.ts'), 'utf8'), 'export const b = 3;\n'); // edit_many takes two spellings of one file as that file, planned once // from one text, and names it as disk does. @@ -139,7 +139,7 @@ test('a read, write, edit and delete spelled in another case act on disk\'s file { path: 'work/SRC/B.TS', oldString: 'b', newString: 'c' }, { path: 'work/src/b.ts', oldString: '3', newString: '4' }, ] }); - assert.deepEqual((many.data as { files: Array<{ path: string; edits: number[] }> }).files.map((f) => [f.path, f.edits]), [['work/src/b.ts', [1, 2]]]); + assert.deepEqual((many.structured as { files: Array<{ path: string; edits: number[] }> }).files.map((f) => [f.path, f.edits]), [['work/src/b.ts', [1, 2]]]); assert.equal(readFileSync(join(dir, 'src', 'b.ts'), 'utf8'), 'export const c = 4;\n'); const listed = await call('ls', { path: 'work', recursive: true }); @@ -235,21 +235,21 @@ test('a name with ß, spelled in another case, takes disk\'s spelling', { skip: // ß's upper case is two letters, SS, which a disk whose folding is simple doesn't fold to ß. writeFileSync(join(dir, 'src', 'stra\u00dfe.txt'), 'sharp s\n'); const read = await call('read', { path: 'work/src/Stra\u00dfe.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/stra\u00dfe.txt'); + assert.equal((read.structured as { path: string }).path, 'work/src/stra\u00dfe.txt'); }); test('where disk folds ß with ss, a name spelled with ss takes disk\'s ß', { skip: SHARP_S_FOLDING_ONLY }, async (t) => { const { dir, call } = await setup(t); writeFileSync(join(dir, 'src', 'stra\u00dfe.txt'), 'sharp s\n'); const read = await call('read', { path: 'work/src/STRASSE.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/stra\u00dfe.txt'); + assert.equal((read.structured as { path: string }).path, 'work/src/stra\u00dfe.txt'); }); test('where disk folds ſ with s, a name spelled with s takes disk\'s ſ', { skip: LONG_S_FOLDING_ONLY }, async (t) => { const { dir, call } = await setup(t); writeFileSync(join(dir, 'src', '\u017foap.txt'), 'long s\n'); const read = await call('read', { path: 'work/src/SOAP.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/\u017foap.txt'); + assert.equal((read.structured as { path: string }).path, 'work/src/\u017foap.txt'); }); test('where disk folds ς with σ, a name spelled with σ takes disk\'s ς', { skip: FINAL_SIGMA_FOLDING_ONLY }, async (t) => { @@ -257,7 +257,7 @@ test('where disk folds ς with σ, a name spelled with σ takes disk\'s ς', { s // Lower case keeps them apart here: ΟΔΟΣ.txt lowers to οδοσ.txt, since a letter follows the dot. writeFileSync(join(dir, 'src', '\u03bf\u03b4\u03bf\u03c2.txt'), 'final sigma\n'); const read = await call('read', { path: 'work/src/\u039f\u0394\u039f\u03a3.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/\u03bf\u03b4\u03bf\u03c2.txt'); + assert.equal((read.structured as { path: string }).path, 'work/src/\u03bf\u03b4\u03bf\u03c2.txt'); }); test('a name with dotless ı, spelled in another case, takes disk\'s spelling', { skip: FOLDING_ONLY }, async (t) => { @@ -265,7 +265,7 @@ test('a name with dotless ı, spelled in another case, takes disk\'s spelling', // ı's upper case, I, lowers to i, which Unicode's case folding keeps apart from ı. writeFileSync(join(dir, 'src', 'k\u0131r.txt'), 'dotless i\n'); const read = await call('read', { path: 'work/src/K\u0131r.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/k\u0131r.txt'); + assert.equal((read.structured as { path: string }).path, 'work/src/k\u0131r.txt'); }); test('the public readers and writer take a path as disk spells it too', { skip: FOLDING_ONLY }, async (t) => { @@ -288,7 +288,7 @@ test('a name in the other Unicode normalization form takes the form disk lists', await call('ls', { path: 'work', recursive: true }); pushed(); const read = await call('read', { path: 'work/\u304c/a.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/\u304b\u3099/a.txt'); + assert.equal((read.structured as { path: string }).path, 'work/\u304b\u3099/a.txt'); await call('ls', { path: 'work', recursive: true }); assert.deepEqual(pushed(), []); }); @@ -363,8 +363,8 @@ test("through a link a mount follows, the include judges where the path lands as assert.ok(String(read.error).endsWith('Symlink escape detected: work/x.json'), String(read.error)); await assert.rejects(workspace.readFileFromDisk('work/x.json'), (err: unknown) => err instanceof WorkspaceReadError && err.code === 'escape'); await call('sync', {}); - assert.equal((((await call('grep', { path: 'work', pattern: 'outside the include' })).data) as { totalMatches: number }).totalMatches, 0); - assert.match(JSON.stringify((await call('read', { path: 'work/w.json' })).data), /covered/); + assert.equal((((await call('grep', { path: 'work', pattern: 'outside the include' })).structured) as { totalMatches: number }).totalMatches, 0); + assert.match(JSON.stringify((await call('read', { path: 'work/w.json' })).structured), /covered/); const listing = JSON.stringify((await call('ls', { path: 'work/via/sub' })).data); assert.ok(!listing.includes('y.txt'), listing); }); @@ -580,7 +580,7 @@ test('on a disk that tells case apart, the tools take a directory spelled otherw ]; }); const read = await call('read', { path: 'outer/Inner/x.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { content: string }).content, 'outer\n'); + assert.equal((read.structured as { content: string }).content, 'outer\n'); // Synchronous, it compares the path as given loosely, as a disk that // folds names may, so it takes Inner for the nested mount's directory. assert.equal(workspace.resolveAbsolutePath('outer/Inner/x.txt'), null); @@ -615,7 +615,7 @@ test('where disk folds case and keeps \u00df apart from ss, the tools take a dir // The tools compare a respelled path exactly, so `strasse` is the outer // mount's own directory, as disk keeps it. const read = await call('read', { path: 'outer/strasse/x.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { content: string }).content, 'outer\n'); + assert.equal((read.structured as { content: string }).content, 'outer\n'); // resolveAbsolutePath folds case as listedName does, loosely too, with no // same-file check, so it errs toward the nested mount and gives null. assert.equal(workspace.resolveAbsolutePath('outer/strasse/x.txt'), null); @@ -778,7 +778,7 @@ test("a walk through a link that spells a directory above a nested mount's other const tool = async (name: string, input: Record): Promise> => { const res = await workspace.handleToolCall({ id: name, name, input } as Parameters[0]); assert.equal(res.success, true, `${name}: ${String(res.error)}`); - return res.data as Record; + return (res.structured ?? res.data) as Record; }; const listing = await tool('ls', { path: 'outer/x/sub', recursive: true }); assert.deepEqual((listing.entries as Array<{ path: string }>).map((e) => e.path), ['x/sub/own.txt']); @@ -814,8 +814,8 @@ test('a second draft spelled in another case is the first one, and a push writes const listed = await call('ls', { path: 'work', recursive: true }); assert.deepEqual((listed.data as { entries: Array<{ path: string }> }).entries.map((e) => e.path).sort(), ['src/New.txt', 'src/a.ts']); const read = await call('read', { path: 'work/SRC/new.TXT', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/New.txt'); - assert.equal((read.data as { content: string }).content, 'second\n'); + assert.equal((read.structured as { path: string }).path, 'work/src/New.txt'); + assert.equal((read.structured as { content: string }).content, 'second\n'); const materialized = await call('materialize', { mount: 'work' }); assert.deepEqual(materialized.data, { materialized: [{ mount: 'work', path: 'src/New.txt' }], count: 1 }); assert.deepEqual(readdirSync(join(dir, 'src')).sort(), ['New.txt', 'a.ts']); @@ -847,8 +847,8 @@ test('a draft that disk lists in another case is reached by its own path, and de // lists that spelling, and the workspace holds the draft under its own. writeFileSync(join(dir, 'src', 'LATE.txt'), 'made on disk\n'); const read = await call('read', { path: 'work/src/Late.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/src/Late.txt'); - assert.equal((read.data as { content: string }).content, 'draft\n'); + assert.equal((read.structured as { path: string }).path, 'work/src/Late.txt'); + assert.equal((read.structured as { content: string }).content, 'draft\n'); const deleted = await call('delete', { path: 'work/src/Late.txt' }); assert.equal((deleted.data as { path: string }).path, 'work/src/Late.txt'); const listed = await call('ls', { path: 'work', recursive: true }); @@ -865,8 +865,8 @@ test('a draft in a directory disk comes to list in another case is still reached mkdirSync(join(dir, 'docs')); writeFileSync(join(dir, 'docs', 'x.txt'), 'made on disk\n'); const read = await call('read', { path: 'work/Docs/a.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((read.data as { path: string }).path, 'work/Docs/a.txt'); - assert.equal((read.data as { content: string }).content, 'draft\n'); + assert.equal((read.structured as { path: string }).path, 'work/Docs/a.txt'); + assert.equal((read.structured as { content: string }).content, 'draft\n'); const again = await call('write', { path: 'work/Docs/a.txt', content: 'again\n' }); assert.equal((again.data as { path: string }).path, 'work/Docs/a.txt'); const listed = await call('ls', { path: 'work', recursive: true }); @@ -910,7 +910,7 @@ test('a file the workspace deleted, which disk then changed and lists in another const status = (await call('status', {})).data as { work: { conflicts: number; pendingWorkspaceDeletions: number } }; assert.deepEqual([status.work.conflicts, status.work.pendingWorkspaceDeletions], [0, 0], 'the sync settled the conflict and the deletion'); const again = await call('read', { path: 'work/src/Name.txt', offsetChars: 0, limitChars: 100 }); - assert.equal((again.data as { content: string }).content, 'changed on disk\n'); + assert.equal((again.structured as { content: string }).content, 'changed on disk\n'); const listed = await call('ls', { path: 'work', recursive: true }); assert.equal((listed.data as { entries: Array<{ path: string }> }).entries.filter((e) => e.path.toLowerCase() === 'src/name.txt').length, 1, 'one entry for the file'); }); diff --git a/test/workspace-read-text.test.ts b/test/workspace-read-text.test.ts index f8b81fb9d..17fe54ad2 100644 --- a/test/workspace-read-text.test.ts +++ b/test/workspace-read-text.test.ts @@ -25,72 +25,84 @@ async function setup(t: TestContext, content: string | Buffer) { type LinePage = { totalLines: number; fromLine?: number; toLine?: number; note?: string; content: string }; type CharPage = { totalChars: number; offsetChars: number; nextOffsetChars: number | null; note?: string; content: string }; -const stored = (result: { data?: unknown }): number => JSON.stringify(result.data).length; +/** The text the model reads of a page: its one text block. */ +const textOf = (result: { data?: unknown }): string => { + const blocks = result.data as Array<{ type: string; text: string }>; + assert.equal(blocks.length, 1); + return blocks[0]!.text; +}; +/** How long a page is as the result limit measures it: its text's length. */ +const stored = (result: { data?: unknown }): number => textOf(result).length; +/** A page's text, as read writes it: its fields, one to a line, then its content under its label. */ +const pageText = (page: Record): string => [ + ...Object.entries(page).filter(([key, value]) => key !== 'content' && value !== undefined).map(([key, value]) => `${key}: ${String(value)}`), + ...(page.content ? [`--- content ---\n${String(page.content)}`] : []), +].join('\n'); // A running script's call's resultMaxChars at the default 128 MiB of script memory. const SCRIPT_LIMIT = 7_456_527; test('line reads retain numbering, ranges, and the 2000-line default', async t => { const { read } = await setup(t, Array.from({ length: 2005 }, (_, i) => `line ${i + 1}`).join('\n')); const result = await read({ offset: 2, limit: 2 }); - assert.deepEqual(result.data, { + assert.deepEqual(result.structured, { path: 'work/text.txt', totalLines: 2005, fromLine: 2, toLine: 3, note: 'Truncated at 2 lines (file has 2005). Use offset/limit to read more.', content: ' 2\tline 2\n 3\tline 3', }); // A script's call can take far more than a turn's, so its default page is the 2000 lines. - const defaults = (await read({}, SCRIPT_LIMIT)).data as LinePage; + const defaults = (await read({}, SCRIPT_LIMIT)).structured as LinePage; assert.equal(defaults.toLine, 2000); assert.equal(defaults.content.split('\n').length, 2000); }); test('a CRLF file reads as its lines without the CR, and says so', async t => { const { read } = await setup(t, 'one\r\ntwo\r\n'); - assert.deepEqual((await read()).data, { + assert.deepEqual((await read()).structured, { path: 'work/text.txt', totalLines: 2, fromLine: 1, toLine: 2, note: 'Line ends are CRLF, shown without the CR. An edit matches text as read shows it, and keeps the CRLF ends.', content: ' 1\tone\n 2\ttwo', }); const { read: readMixed } = await setup(t, 'a\r\nb\nc\r\nd'); - const page = (await readMixed()).data as LinePage; + const page = (await readMixed()).structured as LinePage; assert.equal(page.content, ' 1\ta\n 2\tb\n 3\tc\n 4\td'); assert.equal(page.note, 'Line ends are mixed, 2 CRLF and 1 LF, and a CRLF end is shown without its CR. An edit matches text as read shows it, and keeps each line\'s end.'); // One line whose own text ends in a CR, in a file of mixed ends: the note counts it, in the singular. const { read: readKept } = await setup(t, 'x\r\r\ny\nz\r\n'); - const kept = (await readKept()).data as LinePage; + const kept = (await readKept()).structured as LinePage; assert.equal(kept.content, ' 1\tx\r\r\n 2\ty\n 3\tz'); assert.equal(kept.note, 'Line ends are mixed, 2 CRLF and 1 LF, and a CRLF end is shown without its CR, except on 1 line whose own text ends in a CR, which shows both CRs. An edit matches text as read shows it, and keeps each line\'s end.'); // A CR that doesn't end a line is the line's own. const { read: readCr } = await setup(t, 'a\rb\n'); - assert.deepEqual((await readCr()).data, { path: 'work/text.txt', totalLines: 1, fromLine: 1, toLine: 1, content: ' 1\ta\rb' }); + assert.deepEqual((await readCr()).structured, { path: 'work/text.txt', totalLines: 1, fromLine: 1, toLine: 1, content: ' 1\ta\rb' }); }); test('a BOM is noted and not shown, and bytes that aren\'t UTF-8 are noted', async t => { const { read } = await setup(t, Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from('first\nsecond\n')])); - assert.deepEqual((await read()).data, { + assert.deepEqual((await read()).structured, { path: 'work/text.txt', totalLines: 2, fromLine: 1, toLine: 2, note: 'The file starts with a UTF-8 BOM, which isn\'t shown. An edit keeps it.', content: ' 1\tfirst\n 2\tsecond', }); // Character paging reads the stored text as it is, the BOM included. - assert.equal(((await read({ offsetChars: 0, limitChars: 3 })).data as CharPage).content, '\ufefffi'); + assert.equal(((await read({ offsetChars: 0, limitChars: 3 })).structured as CharPage).content, '\ufefffi'); // A BOM alone is an empty file that has one. const { read: readBomOnly } = await setup(t, Buffer.from([0xef, 0xbb, 0xbf])); - assert.deepEqual((await readBomOnly()).data, { + assert.deepEqual((await readBomOnly()).structured, { path: 'work/text.txt', totalLines: 0, note: 'The file starts with a UTF-8 BOM, which isn\'t shown. An edit keeps it.', content: '', }); const notUtf8 = 'The file isn\'t valid UTF-8: each invalid byte sequence shows as U+FFFD. Edit refuses this file, and write replaces it whole.'; const { read: readLatin1 } = await setup(t, Buffer.from('caf\xe9\n', 'latin1')); - assert.deepEqual((await readLatin1()).data, { + assert.deepEqual((await readLatin1()).structured, { path: 'work/text.txt', totalLines: 1, fromLine: 1, toLine: 1, note: notUtf8, content: ' 1\tcaf\ufffd', }); - assert.equal(((await readLatin1({ offsetChars: 0 })).data as CharPage).note, notUtf8); + assert.equal(((await readLatin1({ offsetChars: 0 })).structured as CharPage).note, notUtf8); }); test('a long line is cut at 2000 characters, with the offsetChars and limitChars that read the rest', async t => { // A BOM and a CRLF before the long line, so its offset counts both. const long = 'x'.repeat(1999) + '😀' + 'y'.repeat(1000); const { read } = await setup(t, Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(`short\r\n${long}\r\nlast\r\n`)])); - const page = (await read({}, 24_000)).data as LinePage; + const page = (await read({}, 24_000)).structured as LinePage; const lines = page.content.split('\n'); assert.equal(lines.length, 3); // The cut doesn't split the emoji's surrogate pair: it falls before it. @@ -98,12 +110,12 @@ test('a long line is cut at 2000 characters, with the offsetChars and limitChars const at = 1 + 'short\r\n'.length + 1999; assert.equal(lines[1], ` 2\t${'x'.repeat(1999)} [… read the rest with offsetChars=${at} and limitChars=${rest}]`); // A script's call cuts the line the same way. - assert.equal(((await read({}, SCRIPT_LIMIT)).data as LinePage).content.split('\n')[1], lines[1]); - const more = (await read({ offsetChars: at, limitChars: rest })).data as CharPage; + assert.equal(((await read({}, SCRIPT_LIMIT)).structured as LinePage).content.split('\n')[1], lines[1]); + const more = (await read({ offsetChars: at, limitChars: rest })).structured as CharPage; assert.equal(more.content, '😀' + 'y'.repeat(1000)); assert.equal(lines[2], ' 3\tlast'); // Where the cut line ends the page, its note names the offset its marker does. - const two = (await read({ limit: 2 }, 24_000)).data as LinePage; + const two = (await read({ limit: 2 }, 24_000)).structured as LinePage; assert.ok(two.note!.startsWith(`Truncated at 2 lines (file has 3). Line 2 is cut: read its rest with offsetChars=${at} and limitChars=${rest}. Use offset/limit to read more.`), two.note); }); @@ -113,16 +125,16 @@ test('a line page stops before its result would pass the limit, and says where t for (const limit of [24_000, 5_000, 1_000]) { const result = await read({}, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as LinePage; + const page = result.structured as LinePage; assert.ok(page.toLine! < 2000); assert.equal(page.note, `Stopped at line ${page.toLine} to stay within the ${limit}-character result limit (file has 3000). Continue with offset=${page.toLine! + 1}.`); // One more line would have passed the limit. const more = await read({ limit: page.toLine! + 1 }, 5_000_000); - const withNext = { ...(result.data as object), toLine: page.toLine! + 1, content: (more.data as LinePage).content }; - assert.ok(JSON.stringify(withNext).length > limit); + const withNext = { ...(result.structured as object), toLine: page.toLine! + 1, content: (more.structured as LinePage).content }; + assert.ok(pageText(withNext).length > limit); // Continuing reads on from the next line, ends at the line its last shown // line is, and says to go on after it. - const next = (await read({ offset: page.toLine! + 1 }, limit)).data as LinePage; + const next = (await read({ offset: page.toLine! + 1 }, limit)).structured as LinePage; assert.equal(next.fromLine, page.toLine! + 1); assert.ok(next.content.startsWith(`${String(page.toLine! + 1).padStart(6)}\tline ${page.toLine! + 1}:`)); const lastShown = next.content.slice(next.content.lastIndexOf('\n') + 1); @@ -137,18 +149,18 @@ test('a line page is the longest that fits, though the page that ends where it w const { read } = await setup(t, `${'x'.repeat(900)}\na\n`); const both = await read({}, 1_000); assert.ok(stored(both) <= 1_000, `${stored(both)}`); - assert.deepEqual(both.data, { path: 'work/text.txt', totalLines: 2, fromLine: 1, toLine: 2, content: ` 1\t${'x'.repeat(900)}\n 2\ta` }); + assert.deepEqual(both.structured, { path: 'work/text.txt', totalLines: 2, fromLine: 1, toLine: 2, content: ` 1\t${'x'.repeat(900)}\n 2\ta` }); const { read: readThree } = await setup(t, `${'a'.repeat(300)}\n${'b'.repeat(550)}\nc\n`); const three = await readThree({}, 1_000); assert.ok(stored(three) <= 1_000, `${stored(three)}`); - assert.equal((three.data as LinePage).toLine, 3); - assert.equal((three.data as LinePage).note, undefined); + assert.equal((three.structured as LinePage).toLine, 3); + assert.equal((three.structured as LinePage).note, undefined); // The page at the line limit says so in fewer words than a stop, so it fits // at its own length, where the page a line shorter, with a stop note, doesn't. const { read: readLimit } = await setup(t, `${'a'.repeat(800)}\nb\nc\n`); const atLimit = await readLimit({ limit: 2 }, 5_000_000); - assert.equal((atLimit.data as LinePage).note, 'Truncated at 2 lines (file has 3). Use offset/limit to read more.'); - assert.deepEqual((await readLimit({ limit: 2 }, stored(atLimit))).data, atLimit.data); + assert.equal((atLimit.structured as LinePage).note, 'Truncated at 2 lines (file has 3). Use offset/limit to read more.'); + assert.deepEqual((await readLimit({ limit: 2 }, stored(atLimit))).structured, atLimit.structured); // The page that ends at a cut line names that line's rest, in more words // than a page that ends at a whole line: so the page of lines 1 and 2 fits // where the page of line 1 alone, cut at 2,000 characters, doesn't. @@ -158,9 +170,9 @@ test('a line page is the longest that fits, though the page that ends where it w note: `Stopped at line 2 to stay within the ${limit}-character result limit (file has 3). Continue with offset=3.`, content: ` 1\t${'q'.repeat(2000)} [… read the rest with offsetChars=2000 and limitChars=500]\n 2\tz`, }); - const limit = JSON.stringify(two(9_999)).length; + const limit = pageText(two(9_999)).length; assert.equal(String(limit).length, 4, 'the limit has as many digits as the 9,999 it was measured with'); - assert.deepEqual((await readCut({}, limit)).data, two(limit)); + assert.deepEqual((await readCut({}, limit)).structured, two(limit)); }); test('a page with notes holds to the result limit with its notes', async t => { @@ -169,18 +181,18 @@ test('a page with notes holds to the result limit with its notes', async t => { for (const limit of [1_000, 1_500]) { const result = await read({}, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as LinePage; + const page = result.structured as LinePage; assert.equal(page.note, `Stopped at line ${page.toLine} to stay within the ${limit}-character result limit (file has 300). Continue with offset=${page.toLine! + 1}. Line ends are CRLF, shown without the CR. An edit matches text as read shows it, and keeps the CRLF ends.`); // One more line would have passed the limit. const more = await read({ limit: page.toLine! + 1 }, 5_000_000); - assert.ok(JSON.stringify({ ...page, toLine: page.toLine! + 1, content: (more.data as LinePage).content }).length > limit); + assert.ok(pageText({ ...page, toLine: page.toLine! + 1, content: (more.structured as LinePage).content }).length > limit); } // An invalid-UTF-8 file's note shares a character page's room. const { read: readBad } = await setup(t, Buffer.concat([Buffer.from('a'.repeat(5000)), Buffer.from([0xff]), Buffer.from('b'.repeat(5000))])); for (const limit of [1_000, 1_500]) { const result = await readBad({ offsetChars: 0, limitChars: 100_000 }, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as CharPage; + const page = result.structured as CharPage; assert.equal(page.note, `Stopped at character ${page.nextOffsetChars} to stay within the ${limit}-character result limit; read the rest of what you asked for with offsetChars=${page.nextOffsetChars} and limitChars=${page.totalChars - page.nextOffsetChars!}. The file isn't valid UTF-8: each invalid byte sequence shows as U+FFFD. Edit refuses this file, and write replaces it whole.`); } }); @@ -194,18 +206,17 @@ test('a generated file reads under the call\'s result limit too', async t => { }); const limited = await read(1_000); assert.ok(stored(limited) <= 1_000, `${stored(limited)}`); - const page = limited.data as LinePage; + const page = limited.structured as LinePage; assert.equal(page.note, `Stopped at line ${page.toLine} to stay within the 1000-character result limit (file has 500). Continue with offset=${page.toLine! + 1}.`); // A call made in process takes it whole. - assert.equal(((await read()).data as LinePage).toLine, 500); + assert.equal(((await read()).structured as LinePage).toLine, 500); }); test('a first line that alone would pass the limit is cut where it fits', async t => { - // Each quote takes two characters as JSON. const { read } = await setup(t, '"'.repeat(1500) + '\nnext\n'); const result = await read({}, 1_000); assert.ok(stored(result) <= 1_000, `${stored(result)}`); - const page = result.data as LinePage; + const page = result.structured as LinePage; assert.equal(page.toLine, 1); const [, shown, at, rest] = /^ {5}1\t("*) \[… read the rest with offsetChars=(\d+) and limitChars=(\d+)\]$/.exec(page.content)!; assert.ok(shown.length > 0); @@ -217,7 +228,7 @@ test('a first line that alone would pass the limit is cut where it fits', async // limit, on a page that goes on and on one that ends the file. const oneMore = (cutPage: LinePage, chars: number) => { const more = chars + 1; - return JSON.stringify({ + return pageText({ ...cutPage, ...(cutPage.note !== undefined ? { note: cutPage.note.replace(/offsetChars=\d+ and limitChars=\d+/, `offsetChars=${more} and limitChars=${1500 - more}`) } : {}), content: ` 1\t${'"'.repeat(more)} [… read the rest with offsetChars=${more} and limitChars=${1500 - more}]`, @@ -227,7 +238,7 @@ test('a first line that alone would pass the limit is cut where it fits', async const { read: readOneLine } = await setup(t, '"'.repeat(1500)); const last = await readOneLine({}, 1_000); assert.ok(stored(last) <= 1_000, `${stored(last)}`); - const lastPage = last.data as LinePage; + const lastPage = last.structured as LinePage; const lastShown = /^ {5}1\t("*) \[/.exec(lastPage.content)![1].length; // The page ends the file, and its last line is cut, so its note names the rest. assert.equal(lastPage.toLine, lastPage.totalLines); @@ -238,9 +249,9 @@ test('a first line that alone would pass the limit is cut where it fits', async const { read: readFits } = await setup(t, 'x'.repeat(900)); const fitting = await readFits({}, 1_000); assert.ok(stored(fitting) <= 1_000, `${stored(fitting)}`); - assert.deepEqual(fitting.data, { path: 'work/text.txt', totalLines: 1, fromLine: 1, toLine: 1, content: ` 1\t${'x'.repeat(900)}` }); + assert.deepEqual(fitting.structured, { path: 'work/text.txt', totalLines: 1, fromLine: 1, toLine: 1, content: ` 1\t${'x'.repeat(900)}` }); // A limit with no room beside the page's own fields still shows where the line's text is. - const tiny = (await read({}, 50)).data as LinePage; + const tiny = (await read({}, 50)).structured as LinePage; assert.equal(tiny.toLine, 1); assert.equal(tiny.content, ' 1\t [… read the rest with offsetChars=0 and limitChars=1500]'); assert.match(tiny.note!, /Line 1 is cut: read its rest with offsetChars=0 and limitChars=1500, then continue with offset=2\.$/); @@ -248,12 +259,12 @@ test('a first line that alone would pass the limit is cut where it fits', async test("a first line cut to fit stays within 2,000 characters, though a longer cut would fit once its rest loses a digit", async t => { const { read } = await setup(t, 'a'.repeat(3_000)); - const withCut = (page: LinePage, cut: number) => JSON.stringify({ + const withCut = (page: LinePage, cut: number) => pageText({ ...page, note: `Line 1 is cut: read its rest with offsetChars=${cut} and limitChars=${3_000 - cut}.`, content: ` 1\t${'a'.repeat(cut)} [… read the rest with offsetChars=${cut} and limitChars=${3_000 - cut}]`, }).length; - const shape = (await read({}, 24_000)).data as LinePage; + const shape = (await read({}, 24_000)).structured as LinePage; // At this limit the cut at 2,000 is a character too long, and the cut at // 2,001 would fit, since its rest's length, 999, has a digit fewer than // 1,000, in the marker and in the note. @@ -261,7 +272,7 @@ test("a first line cut to fit stays within 2,000 characters, though a longer cut assert.equal(withCut(shape, 2_001), limit); const result = await read({}, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as LinePage; + const page = result.structured as LinePage; assert.equal(/^ {5}1\t(a*) \[/.exec(page.content)![1].length, 1_999); assert.equal(stored(result), withCut(page, 1_999)); }); @@ -269,10 +280,10 @@ test("a first line cut to fit stays within 2,000 characters, though a longer cut test("a first line cut to fit is the longest cut that fits, where the rest's length loses a digit too", async t => { // The rest's length is in the marker and in the note, so at the cut where it // goes from 1,000 to 999 the page loses two characters for the one it gains, - // and is a character shorter than at the cut before. Plain letters take one - // character each as JSON, so some limit in this range lands on that dip. + // and is a character shorter than at the cut before. Each letter takes one + // character of the page's text, so some limit in this range lands on that dip. const { read } = await setup(t, 'a'.repeat(1_500)); - const withCut = (page: LinePage, cut: number) => JSON.stringify({ + const withCut = (page: LinePage, cut: number) => pageText({ ...page, note: `Line 1 is cut: read its rest with offsetChars=${cut} and limitChars=${1_500 - cut}.`, content: ` 1\t${'a'.repeat(cut)} [… read the rest with offsetChars=${cut} and limitChars=${1_500 - cut}]`, @@ -281,7 +292,7 @@ test("a first line cut to fit is the longest cut that fits, where the rest's len for (let limit = 700; limit <= 760; limit++) { const result = await read({}, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as LinePage; + const page = result.structured as LinePage; const cut = /^ {5}1\t(a*) \[/.exec(page.content)![1].length; cuts.add(cut); assert.equal(withCut(page, cut), stored(result)); @@ -294,7 +305,7 @@ test("a first line cut to fit is the longest cut that fits, where the rest's len test('a first line cut to fit, read from a later offset, is numbered and placed in the file', async t => { const { read } = await setup(t, `a\nb\n${'"'.repeat(1500)}\nnext\n`); - const page = (await read({ offset: 3 }, 1_000)).data as LinePage; + const page = (await read({ offset: 3 }, 1_000)).structured as LinePage; assert.deepEqual([page.fromLine, page.toLine], [3, 3]); const [, shown, at, rest] = /^ {5}3\t("*) \[… read the rest with offsetChars=(\d+) and limitChars=(\d+)\]$/.exec(page.content)!; assert.ok(shown.length > 0 && shown.length < 1500, `${shown.length}`); @@ -302,7 +313,7 @@ test('a first line cut to fit, read from a later offset, is numbered and placed assert.equal(Number(at), 4 + shown.length); assert.equal(page.note, `Stopped at line 3 to stay within the 1000-character result limit (file has 4). Line 3 is cut: read its rest with offsetChars=${at} and limitChars=${rest}, then continue with offset=4.`); // The rest read with both reads to the line's end and no further. - const more = (await read({ offsetChars: Number(at), limitChars: Number(rest) }, 24_000)).data as CharPage; + const more = (await read({ offsetChars: Number(at), limitChars: Number(rest) }, 24_000)).structured as CharPage; assert.equal(more.content, '"'.repeat(Number(rest))); assert.equal(more.nextOffsetChars, 4 + 1500); }); @@ -314,7 +325,7 @@ test("a cut line's rest read through several character pages, following only the const long = 'z'.repeat(5_000); const { read } = await setup(t, `head\n${long}\n${Array.from({ length: 50 }, (_, i) => `after ${i}`).join('\n')}\n`); // From line 2, the line is cut to fit the page. - const linePage = (await read({ offset: 2 }, 1_000)).data as LinePage; + const linePage = (await read({ offset: 2 }, 1_000)).structured as LinePage; const [, at, left] = /Line 2 is cut: read its rest with offsetChars=(\d+) and limitChars=(\d+), then continue with offset=3\.$/.exec(linePage.note!)!; const lineEnd = 'head\n'.length + long.length; assert.equal(Number(at) + Number(left), lineEnd); @@ -326,7 +337,7 @@ test("a cut line's rest read through several character pages, following only the assert.ok(++pages < 100, 'every page moves on'); const result = await read({ offsetChars, limitChars }, 1_000); assert.ok(stored(result) <= 1_000, `${stored(result)}`); - const page = result.data as CharPage; + const page = result.structured as CharPage; rest += page.content; const next = page.note === undefined ? null : /read the rest of what you asked for with offsetChars=(\d+) and limitChars=(\d+)/.exec(page.note); if (next === null) { @@ -348,22 +359,22 @@ test('a page whose last line is cut names where its rest starts before the next const { read } = await setup(t, `short\n${long}\n${long}\n${long}\n`); const at = (line: number) => 'short\n'.length + (line - 2) * (long.length + 1) + 2000; // The limit stops the page at the cut line. - const byLimit = (await read({ limit: 2 }, 24_000)).data as LinePage; + const byLimit = (await read({ limit: 2 }, 24_000)).structured as LinePage; assert.equal(byLimit.toLine, 2); assert.equal(byLimit.note, `Truncated at 2 lines (file has 4). Line 2 is cut: read its rest with offsetChars=${at(2)} and limitChars=1000. Use offset/limit to read more.`); // The result limit stops it there: line 4 would pass 5,000 characters. - const byBudget = (await read({}, 5_000)).data as LinePage; + const byBudget = (await read({}, 5_000)).structured as LinePage; assert.equal(byBudget.toLine, 3); assert.equal(byBudget.note, `Stopped at line 3 to stay within the 5000-character result limit (file has 4). Line 3 is cut: read its rest with offsetChars=${at(3)} and limitChars=1000, then continue with offset=4.`); // A cut line inside a page has its rest named by its own marker alone. The // page that ends the file names its cut last line too: toLine is the last // line, but the file isn't read whole until that line's rest is. - const whole = (await read({}, 24_000)).data as LinePage; + const whole = (await read({}, 24_000)).structured as LinePage; assert.equal(whole.toLine, 4); assert.equal(whole.note, `Line 4 is cut: read its rest with offsetChars=${at(4)} and limitChars=1000.`); // A page that ends the file on a line shown whole has nothing to say. const { read: readShort } = await setup(t, `${long}\nshort\n`); - assert.equal(((await readShort({}, 24_000)).data as LinePage).note, undefined); + assert.equal(((await readShort({}, 24_000)).structured as LinePage).note, undefined); }); test('a call that carries no resultMaxChars pages by its line limit alone, with every line whole', async t => { @@ -372,27 +383,27 @@ test('a call that carries no resultMaxChars pages by its line limit alone, with const long = 'q'.repeat(3000); const lines = Array.from({ length: 3000 }, (_, i) => (i === 10 ? long : `line ${i + 1} of a file long enough to pass 24,000 characters`)); const { read } = await setup(t, lines.join('\r\n')); - const page = (await read({ limit: 2500 })).data as LinePage; + const page = (await read({ limit: 2500 })).structured as LinePage; assert.equal(page.toLine, 2500); assert.equal(page.note, 'Truncated at 2500 lines (file has 3000). Use offset/limit to read more. Line ends are CRLF, shown without the CR. An edit matches text as read shows it, and keeps the CRLF ends.'); const shown = page.content.split('\n'); assert.equal(shown.length, 2500); assert.equal(shown[10], ` 11\t${long}`); - assert.ok(JSON.stringify(page).length > 100_000, `${JSON.stringify(page).length}`); + assert.ok(pageText(page).length > 100_000, `${pageText(page).length}`); // The default is 2000 lines, and the page that ends the file has no note but the text's. - assert.equal(((await read()).data as LinePage).toLine, 2000); - const end = (await read({ offset: 2001 })).data as LinePage; + assert.equal(((await read()).structured as LinePage).toLine, 2000); + const end = (await read({ offset: 2001 })).structured as LinePage; assert.equal(end.toLine, 3000); assert.equal(end.note, 'Line ends are CRLF, shown without the CR. An edit matches text as read shows it, and keeps the CRLF ends.'); // A character page gives its limitChars whole too. - const chars = (await read({ offsetChars: 0, limitChars: 100_000 })).data as CharPage; + const chars = (await read({ offsetChars: 0, limitChars: 100_000 })).structured as CharPage; assert.equal(chars.content.length, 100_000); assert.equal(chars.nextOffsetChars, 100_000); assert.equal(chars.note, undefined); }); test('a character page stops at the result limit too, with nextOffsetChars', async t => { - // Control characters take six characters each as JSON. + // A control character, which JSON wrote six characters long, takes one in the page's text. const text = 'abc\u0001'.repeat(20_000); const { read } = await setup(t, text); for (const limit of [24_000, 1_000]) { @@ -404,7 +415,7 @@ test('a character page stops at the result limit too, with nextOffsetChars', asy assert.ok(pages < 1_000, 'every page moves on'); const result = await read({ offsetChars, limitChars }, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as CharPage; + const page = result.structured as CharPage; recovered += page.content; if (page.nextOffsetChars === null) break; // What's left of what was asked for is the rest of the text. @@ -415,7 +426,7 @@ test('a character page stops at the result limit too, with nextOffsetChars', asy assert.equal(recovered, text); } // A limit with no room beside the page's own fields still moves the page on. - const tiny = (await read({ offsetChars: 5, limitChars: 100 }, 50)).data as CharPage; + const tiny = (await read({ offsetChars: 5, limitChars: 100 }, 50)).structured as CharPage; assert.equal(tiny.offsetChars, 5); assert.equal(tiny.nextOffsetChars, 6); assert.equal(tiny.content, text[5]); @@ -424,7 +435,7 @@ test('a character page stops at the result limit too, with nextOffsetChars', asy for (let limit = 1_000; limit < 1_004; limit++) { const result = await readEmoji({ offsetChars: 0, limitChars: 100_000 }, limit); assert.ok(stored(result) <= limit, `${stored(result)} > ${limit}`); - const page = result.data as CharPage; + const page = result.structured as CharPage; assert.equal(page.nextOffsetChars! % 2, 0, `at ${limit}`); assert.equal(page.content.length % 2, 0, `at ${limit}`); } @@ -434,33 +445,33 @@ test('a character page moved on by a whole character keeps a surrogate pair whol // A limit too small for the page's own fields still moves the page on, by a // whole character: an emoji's two code units. const { read } = await setup(t, '😀abc'); - assert.deepEqual((await read({ offsetChars: 0, limitChars: 100 }, 50)).data, { + assert.deepEqual((await read({ offsetChars: 0, limitChars: 100 }, 50)).structured, { path: 'work/text.txt', totalChars: 5, offsetChars: 0, nextOffsetChars: 2, note: 'Stopped at character 2 to stay within the 50-character result limit; read the rest of what you asked for with offsetChars=2 and limitChars=3.', content: '😀', }); // Where that step reaches what the page was asked for, the page holds all of it. const { read: readOne } = await setup(t, 'a'); - assert.deepEqual((await readOne({ offsetChars: 0, limitChars: 1 }, 50)).data, { + assert.deepEqual((await readOne({ offsetChars: 0, limitChars: 1 }, 50)).structured, { path: 'work/text.txt', totalChars: 1, offsetChars: 0, nextOffsetChars: null, content: 'a', }); const { read: readSix } = await setup(t, 'abcdef'); - assert.deepEqual((await readSix({ offsetChars: 0, limitChars: 1 }, 50)).data, { + assert.deepEqual((await readSix({ offsetChars: 0, limitChars: 1 }, 50)).structured, { path: 'work/text.txt', totalChars: 6, offsetChars: 0, nextOffsetChars: 1, content: 'a', }); }); test('character mode can be selected by either parameter and reports EOF', async t => { const { read } = await setup(t, 'a'.repeat(3000)); - assert.deepEqual((await read({ offsetChars: 0 })).data, { + assert.deepEqual((await read({ offsetChars: 0 })).structured, { path: 'work/text.txt', totalChars: 3000, offsetChars: 0, nextOffsetChars: 2000, content: 'a'.repeat(2000), }); - assert.equal(((await read({ limitChars: 3 })).data as { content: string }).content, 'aaa'); - assert.deepEqual((await read({ offsetChars: 2999, limitChars: 10 })).data, { + assert.equal(((await read({ limitChars: 3 })).structured as { content: string }).content, 'aaa'); + assert.deepEqual((await read({ offsetChars: 2999, limitChars: 10 })).structured, { path: 'work/text.txt', totalChars: 3000, offsetChars: 2999, nextOffsetChars: null, content: 'a', }); for (const offsetChars of [3000, 4000, Number.MAX_SAFE_INTEGER]) { - assert.deepEqual((await read({ offsetChars })).data, { + assert.deepEqual((await read({ offsetChars })).structured, { path: 'work/text.txt', totalChars: 3000, offsetChars: 3000, nextOffsetChars: null, content: '', }); } @@ -474,7 +485,7 @@ test('character pages preserve Unicode and raw newlines, even with limitChars 1' do { const result = await read({ offsetChars, limitChars: 1 }); assert.equal(result.success, true); - const page = result.data as { content: string; nextOffsetChars: number | null }; + const page = result.structured as { content: string; nextOffsetChars: number | null }; assert.ok(page.content.length >= 1 && page.content.length <= 2); assert.equal(Buffer.from(page.content).toString(), page.content, 'UTF-8 transport cannot replace half a pair'); recovered += page.content; @@ -490,7 +501,7 @@ test('character pages preserve Unicode and raw newlines, even with limitChars 1' test('empty text returns an empty final character page', async t => { const { read } = await setup(t, ''); - assert.deepEqual((await read({ offsetChars: 0 })).data, { + assert.deepEqual((await read({ offsetChars: 0 })).structured, { path: 'work/text.txt', totalChars: 0, offsetChars: 0, nextOffsetChars: null, content: '', }); }); diff --git a/test/workspace-result-text.test.ts b/test/workspace-result-text.test.ts new file mode 100644 index 000000000..c8ad9e6ac --- /dev/null +++ b/test/workspace-result-text.test.ts @@ -0,0 +1,351 @@ +/** + * What a model reads of read's, grep's, edit's and edit_many's answers: their + * text (result-text.ts), a header of the answer's fields and its labelled + * sections, with the fields themselves beside it for a script. A page or a + * receipt is held to the call's result limit as that text measures it, and a + * value holding a line break is written as a JSON string, so it can't start a + * line that reads as the answer's own. + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { JsStore } from '@animalabs/chronicle'; +import { AgentFramework } from '../src/index.js'; +import { WorkspaceModule } from '../src/modules/workspace/index.js'; +import { shownValue } from '../src/modules/workspace/result-text.js'; +import type { EventResponse, Module, ModuleContext } from '../src/types/module.js'; +import type { ProcessEvent, ToolResult } from '../src/types/events.js'; +import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; + +const ONE_PX_PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', + 'base64', +); + +interface Env { + /** A tool's answer, which must succeed, with the call's result limit if given. */ + call(name: string, input: Record, resultMaxChars?: number): Promise; + /** A file written to the mount's directory on disk, which the workspace hasn't stored. */ + disk(path: string, content: string | Buffer): void; + /** A file the workspace stores and disk holds alike. */ + synced(path: string, content: string): Promise; +} + +/** A workspace with a mount `work`, and, where asked, a mount `inner` in its directory `inner`. */ +async function withWorkspace(fn: (env: Env) => Promise, opts: { nested?: boolean } = {}): Promise { + const root = mkdtempSync(join(tmpdir(), 'workspace-result-text-')); + const dir = join(root, 'work'); + mkdirSync(join(dir, 'inner'), { recursive: true }); + const store = JsStore.openOrCreate({ path: join(root, 'ws.chronicle') }); + const workspace = new WorkspaceModule({ mounts: [ + { name: 'work', path: dir, mode: 'read-write', watch: 'never' }, + ...(opts.nested ? [{ name: 'inner', path: join(dir, 'inner'), mode: 'read-write' as const, watch: 'never' as const }] : []), + ] }); + workspace.initStore(store); + await workspace.start({ isRestart: false, getState: () => null, setState: () => {}, pushEvent: () => {} } as unknown as ModuleContext); + let n = 0; + const call = async (name: string, input: Record, resultMaxChars?: number) => { + const result = await workspace.handleToolCall({ id: `call-${n++}`, name, input, ...(resultMaxChars !== undefined ? { resultMaxChars } : {}) }); + assert.equal(result.success, true, `${name}: ${result.error}`); + return result; + }; + const disk = (path: string, content: string | Buffer) => { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + writeFileSync(join(dir, path), content); + }; + try { + await fn({ + call, + disk, + synced: async (path, content) => { + await call('write', { path: `work/${path}`, content }); + await call('materialize', { path: `work/${path}` }); + }, + }); + } finally { + await workspace.stop().catch(() => {}); + if (!store.isClosed()) store.close(); + rmSync(root, { recursive: true, force: true }); + } +} + +/** The text an answer gives the model: its one text block. */ +function textOf(result: ToolResult): string { + assert.ok(Array.isArray(result.data), 'the answer is a list of blocks'); + assert.equal(result.data.length, 1, 'one block'); + const [block] = result.data as Array<{ type: string; text: string }>; + assert.equal(block!.type, 'text'); + return block!.text; +} + +const fields = (result: ToolResult): T => result.structured as T; + +test('read answers a line page as its fields, one to a line, and its numbered lines under content', async () => { + await withWorkspace(async ({ call, disk }) => { + disk('a.txt', 'alpha\nbeta\ngamma\n'); + const page = await call('read', { path: 'work/a.txt' }, 24_000); + assert.equal(textOf(page), 'path: work/a.txt\ntotalLines: 3\nfromLine: 1\ntoLine: 3\n--- content ---\n 1\talpha\n 2\tbeta\n 3\tgamma'); + assert.deepEqual(fields(page), { path: 'work/a.txt', totalLines: 3, fromLine: 1, toLine: 3, content: ' 1\talpha\n 2\tbeta\n 3\tgamma' }); + + disk('crlf.txt', 'one\r\ntwo\r\n'); + const crlf = await call('read', { path: 'work/crlf.txt', offset: 2 }, 24_000); + assert.equal(textOf(crlf), 'path: work/crlf.txt\ntotalLines: 2\nfromLine: 2\ntoLine: 2\n' + + 'note: Line ends are CRLF, shown without the CR. An edit matches text as read shows it, and keeps the CRLF ends.\n' + + '--- content ---\n 2\ttwo'); + + // A page with no lines has no content section. + disk('empty.txt', ''); + assert.equal(textOf(await call('read', { path: 'work/empty.txt' }, 24_000)), 'path: work/empty.txt\ntotalLines: 0'); + }); +}); + +test("a character page names nextOffsetChars, null at the file's end, and shows the file's characters as they are", async () => { + await withWorkspace(async ({ call, disk }) => { + const raw = 'one\r\ntwo \u001b[1mbold\u001b[0m\r\n'; + disk('log.txt', raw); + const first = await call('read', { path: 'work/log.txt', offsetChars: 0, limitChars: 5 }, 24_000); + assert.equal(textOf(first), `path: work/log.txt\ntotalChars: ${raw.length}\noffsetChars: 0\nnextOffsetChars: 5\n--- content ---\none\r\n`); + // A control character reaches the model as itself, where JSON wrote an escape. + const rest = await call('read', { path: 'work/log.txt', offsetChars: 5, limitChars: 100 }, 24_000); + assert.equal(textOf(rest), `path: work/log.txt\ntotalChars: ${raw.length}\noffsetChars: 5\nnextOffsetChars: null\n--- content ---\n${raw.slice(5)}`); + assert.deepEqual(fields(rest), { path: 'work/log.txt', totalChars: raw.length, offsetChars: 5, nextOffsetChars: null, content: raw.slice(5) }); + // Past the end, the page is its fields alone. + const end = await call('read', { path: 'work/log.txt', offsetChars: raw.length }, 24_000); + assert.equal(textOf(end), `path: work/log.txt\ntotalChars: ${raw.length}\noffsetChars: ${raw.length}\nnextOffsetChars: null`); + }); +}); + +test("a line page is the longest whose text fits the call's limit, however JSON would write its lines", async () => { + await withWorkspace(async ({ call, disk }) => { + // Quotes and backslashes, which JSON writes as two characters each. + const lines = Array.from({ length: 400 }, (_, i) => `line ${i + 1} "${'\\"'.repeat(20)}"`); + disk('quoted.txt', `${lines.join('\n')}\n`); + for (const limit of [2_000, 2_001, 5_000]) { + const page = await call('read', { path: 'work/quoted.txt' }, limit); + const text = textOf(page); + const { toLine } = fields<{ toLine: number }>(page); + assert.ok(text.length <= limit, `the page's text, ${text.length}, fits ${limit}`); + assert.ok(JSON.stringify(fields(page)).length > limit, 'as JSON the same page would pass the limit'); + // The next line, after its line feed, with the stop note's two numbers a + // digit longer at most, would pass it. + const next = `${String(toLine + 1).padStart(6)}\t${lines[toLine]}`; + assert.ok(text.length + 1 + next.length > limit - 2, `line ${toLine + 1} doesn't fit beside the page's ${text.length}`); + assert.match(text, new RegExp(`note: Stopped at line ${toLine} to stay within the ${limit}-character result limit \\(file has 400\\)\\. Continue with offset=${toLine + 1}\\.\n--- content ---\n`)); + } + }); +}); + +test("a character page stops where its text reaches the call's limit, and never inside a surrogate pair", async () => { + await withWorkspace(async ({ call, disk }) => { + const quoted = '"\\'.repeat(1_500); + disk('quoted.txt', quoted); + const page = await call('read', { path: 'work/quoted.txt', offsetChars: 0, limitChars: 3_000 }, 1_000); + const text = textOf(page); + assert.ok(text.length <= 1_000 && text.length >= 995, `the page's text, ${text.length}, fills the limit of 1,000`); + const { nextOffsetChars } = fields<{ nextOffsetChars: number }>(page); + assert.match(text, new RegExp(`^note: Stopped at character ${nextOffsetChars} to stay within the 1000-character result limit; read the rest of what you asked for with offsetChars=${nextOffsetChars} and limitChars=${3_000 - nextOffsetChars}\\.$`, 'm')); + + // Every other code unit starts a pair: at one of two limits a unit apart, + // the page's end falls inside one, and the page stops before it. + disk('emoji.txt', '😀'.repeat(1_500)); + for (const limit of [1_000, 1_001]) { + const emoji = await call('read', { path: 'work/emoji.txt', offsetChars: 0, limitChars: 3_000 }, limit); + const { content, nextOffsetChars: next } = fields<{ content: string; nextOffsetChars: number }>(emoji); + assert.equal(next % 2, 0, `at ${limit}, the page ends between pairs`); + assert.equal(content, '😀'.repeat(next / 2)); + assert.ok(textOf(emoji).length <= limit && textOf(emoji).length >= limit - 6); + } + }); +}); + +test("grep answers each version of a file it matched as a section, a match as N: and its context as N-, then what it skipped", async () => { + await withWorkspace(async ({ call, disk, synced }) => { + await synced('a.txt', 'needle one\nhay\nneedle again'); + await synced('b.txt', 'nothing'); + disk('b.txt', 'needle on disk'); + await call('write', { path: 'work/b.txt', content: 'needle in the workspace' }); + await synced('c.txt', 'x\r\nneedle crlf\r\n'); + disk('shot.png', ONE_PX_PNG); + const found = await call('grep', { pattern: 'needle' }, 24_000); + assert.equal(textOf(found), [ + 'pattern: needle', + 'totalMatches: 5', + '--- work/a.txt: workspace (synced) ---', + ' 1:needle one', + ' 3:needle again', + '--- work/b.txt: workspace (conflict) ---', + ' 1:needle in the workspace', + '--- work/b.txt: conflicting-disk (conflict) ---', + ' 1:needle on disk', + '--- work/c.txt: workspace (synced); lineEndings: CRLF ---', + ' 2:needle crlf', + '--- skipped ---', + 'work/shot.png: disk-only: binary or over the size limit, never stored (70 bytes, image/png)', + ].join('\n')); + assert.deepEqual(fields<{ results: Array<{ file: string; version: string }> }>(found).results.map((r) => `${r.file} ${r.version}`), + ['work/a.txt workspace', 'work/b.txt workspace', 'work/b.txt conflicting-disk', 'work/c.txt workspace']); + + // Context starts at the file's first line at the earliest, and `--` parts one match's lines from the next's. + const context = await call('grep', { pattern: 'needle', path: 'work/a.txt', contextBefore: 1 }, 24_000); + assert.equal(textOf(context), [ + 'pattern: needle', + 'totalMatches: 2', + '--- work/a.txt: workspace (synced) ---', + ' 1:needle one', + '--', + ' 2-hay', + ' 3:needle again', + ].join('\n')); + const after = await call('grep', { pattern: 'one', path: 'work/a.txt', contextAfter: 1 }, 24_000); + assert.match(textOf(after), /\n {5}1:needle one\n {5}2-hay$/); + + // A pattern that matches nothing is its header alone, with a note where it has one. + assert.equal(textOf(await call('grep', { pattern: 'no such text', path: 'work/a.txt' }, 24_000)), 'pattern: no such text\ntotalMatches: 0'); + assert.match(textOf(await call('grep', { pattern: 'crlf\\r', path: 'work/c.txt' }, 24_000)), + /^pattern: crlf\\r\ntotalMatches: 0\nnote: The pattern names a CR, and 1 file searched has CRLF line ends: [^\n]*$/); + }); +}); + +test("grep's answer ends with the places it couldn't search, each with why", async () => { + await withWorkspace(async ({ call }) => { + const answer = await call('grep', { pattern: 'needle', path: 'work/inner' }, 24_000); + const { incomplete } = fields<{ incomplete: Array<{ path: string; reason: string }> }>(answer); + assert.equal(incomplete.length, 1); + assert.equal(textOf(answer), `pattern: needle\ntotalMatches: 0\n--- incomplete ---\nwork/inner: ${incomplete[0]!.reason}`); + }, { nested: true }); +}); + +test('edit and edit_many answer their counts and notes, the files edit_many changed, and the changed regions last', async () => { + await withWorkspace(async ({ call, synced }) => { + await synced('a.txt', 'one\ntwo\nthree\nfour\nfive\nsix\n'); + await synced('b.txt', 'alpha\r\nbeta\r\n'); + const edit = await call('edit', { path: 'work/a.txt', oldString: 'four', newString: 'FOUR' }, 24_000); + const done = fields<{ size: number; changed: string }>(edit); + assert.equal(done.changed, ' 2\ttwo\n 3\tthree\n 4\tFOUR\n 5\tfive\n 6\tsix'); + assert.equal(textOf(edit), `path: work/a.txt\nsize: ${done.size}\nreplacements: 1\n--- changed ---\n${done.changed}`); + + const many = await call('edit_many', { edits: [ + { path: 'work/a.txt', oldString: 'one', newString: 'ONE' }, + { path: 'work/b.txt', oldString: 'beta', newString: 'BETA' }, + ] }, 24_000); + const both = fields<{ files: Array<{ size: number }>; changed: string }>(many); + assert.equal(both.changed, 'work/a.txt\n 1\tONE\n 2\ttwo\n 3\tthree\n\nwork/b.txt\n 1\talpha\n 2\tBETA'); + assert.equal(textOf(many), [ + 'edits: 2', + 'replacements: 2', + '--- files ---', + `work/a.txt: size ${both.files[0]!.size}, replacements 1, edits 1`, + `work/b.txt: size ${both.files[1]!.size}, replacements 1, edits 2. The file's CRLF line ends are kept.`, + '--- changed ---', + both.changed, + ].join('\n')); + }); +}); + +test("an edit's receipt is held to the call's limit as its text measures it", async () => { + await withWorkspace(async ({ call, synced }) => { + // Lines JSON writes at twice their length: the region fits as text, and wouldn't as JSON. + const line = (n: number) => `${n} ${'"\\'.repeat(150)}`; + await synced('q.txt', `${Array.from({ length: 9 }, (_, i) => line(i + 1)).join('\n')}\n`); + const edit = await call('edit', { path: 'work/q.txt', oldString: '5 "', newString: 'five "' }, 2_000); + const text = textOf(edit); + assert.ok(text.length <= 2_000, `the receipt's text, ${text.length}, fits 2,000`); + assert.ok(JSON.stringify(fields(edit)).length > 2_000, 'as JSON the same receipt would pass the limit'); + assert.match(text, /\n {5}3\t3 "\\[^\n]*\n {5}4\t[^\n]*\n {5}5\tfive "[^\n]*\n {5}6\t[^\n]*\n {5}7\t7 "[^\n]*$/); + + // edit_many lists what fits of its files, and counts the rest. + const many = Array.from({ length: 40 }, (_, i) => `file-${String(i).padStart(2, '0')}.txt`); + for (const name of many) await synced(name, 'old\n'); + const listed = await call('edit_many', { edits: many.map((name) => ({ path: `work/${name}`, oldString: 'old', newString: 'new' })) }, 1_000); + const receipt = fields<{ files: unknown[]; filesNotListed: number }>(listed); + assert.ok(receipt.filesNotListed > 0 && receipt.files.length > 0, `${receipt.files.length} listed, ${receipt.filesNotListed} not`); + assert.equal(receipt.files.length + receipt.filesNotListed, 40); + assert.match(textOf(listed), new RegExp(`^edits: 40\nreplacements: 40\nfilesNotListed: ${receipt.filesNotListed}\n--- files ---\n`)); + assert.ok(textOf(listed).length <= 1_000); + }); +}); + +test('a value holding a line break is written as a JSON string, so it stays on its line and gives back the path', async () => { + for (const value of ['a\nb', 'a\rb', 'a\u2028b', 'a\u2029b', '\n']) { + const shown = shownValue(value); + assert.ok(!/[\n\r\u2028\u2029]/.test(shown), `${JSON.stringify(value)} is shown on one line`); + assert.equal(JSON.parse(shown), value); + } + for (const value of ['plain', 'tab\tand \u001b escape', '"quoted"', '']) assert.equal(shownValue(value), value); + assert.equal(shownValue(null), 'null'); + assert.equal(shownValue(12), '12'); + + await withWorkspace(async ({ call, disk, synced }) => { + disk('odd\nname.txt', 'needle\n'); + disk('odd\nshot.png', ONE_PX_PNG); + const quoted = JSON.stringify('work/odd\nname.txt'); + const read = await call('read', { path: 'work/odd\nname.txt' }, 24_000); + assert.equal(textOf(read), `path: ${quoted}\ntotalLines: 1\nfromLine: 1\ntoLine: 1\n--- content ---\n 1\tneedle`); + assert.equal(fields<{ path: string }>(read).path, 'work/odd\nname.txt', 'the fields hold the path as it is'); + const grep = await call('grep', { pattern: 'needle' }, 24_000); + assert.match(textOf(grep), new RegExp(`\n--- ${escape(quoted)}: workspace \\(synced\\) ---\n {5}1:needle\n--- skipped ---\n${escape(JSON.stringify('work/odd\nshot.png'))}: disk-only: `)); + await synced('plain.txt', 'needle\n'); + const many = await call('edit_many', { edits: [ + { path: 'work/odd\nname.txt', oldString: 'needle', newString: 'pin' }, + { path: 'work/plain.txt', oldString: 'needle', newString: 'pin' }, + ] }, 24_000); + const text = textOf(many); + assert.match(text, new RegExp(`\n--- files ---\n${escape(quoted)}: size 4, replacements 1, edits 1\nwork/plain.txt: size 4, replacements 1, edits 2\n`)); + assert.match(text, new RegExp(`\n--- changed ---\n${escape(quoted)}\n {5}1\tpin\n\nwork/plain.txt\n {5}1\tpin$`)); + }); +}); + +const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + +test("through the framework, a model's read reaches it and its history as the page's text, and a script's as the text and the page's fields", async () => { + const root = mkdtempSync(join(tmpdir(), 'workspace-result-text-framework-')); + const dir = join(root, 'files'); + mkdirSync(dir); + writeFileSync(join(dir, 'a.txt'), 'say "hi"\\n\tthere\n'); + const membrane = new MockMembrane(); + membrane.pushResponse(createMockResponse([ + { type: 'tool_use', id: 'call_read', name: 'workspace--read', input: { path: 'files/a.txt' } }, + ], 'tool_use')); + membrane.pushResponse(createMockResponse([{ type: 'text', text: 'Read.' }])); + const workspace = new WorkspaceModule({ mounts: [{ name: 'files', path: dir, mode: 'read-write', watch: 'never' }] }); + // Asks for inference when the message arrives. + const kick: Module = { + name: 'kick', + async start() {}, + async stop() {}, + getTools: () => [], + handleToolCall: async () => ({ success: false, isError: true, error: 'unexpected tool call' }), + onProcess: async (event: ProcessEvent): Promise => (event.type === 'external-message' ? { requestInference: true } : {}), + }; + const framework = await AgentFramework.create({ + storePath: join(root, 'store.chronicle'), membrane: membrane.asMembrane(), + agents: [{ name: 'prime', model: 'mock', systemPrompt: 'test' }], + modules: [workspace, kick], syncIntervalMs: 0, maintenanceIntervalMs: 0, + }); + workspace.initStore(framework.getStore()); + const text = 'path: files/a.txt\ntotalLines: 1\nfromLine: 1\ntoLine: 1\n--- content ---\n 1\tsay "hi"\\n\tthere'; + try { + framework.pushEvent({ type: 'external-message', source: 'test', content: 'read it', metadata: {} } as unknown as ProcessEvent); + await framework.runUntilIdle(); + const messages = (framework.getAgent('prime')?.getContextManager().queryMessages({}).messages ?? []) as unknown as Array<{ content: Array<{ type: string; content?: unknown }> }>; + const stored = messages.flatMap((m) => m.content ?? []).filter((block) => block.type === 'tool_result').map((block) => block.content); + assert.deepEqual(stored, [text], "the history holds the page's text"); + const wire = membrane.lastStream?.receivedToolResults[0] as Array<{ content: unknown }> | undefined; + assert.equal(wire?.[0]?.content, text, 'the model reads the same text'); + + // A script's call is dispatched as the agent's and answered through the running framework. + framework.start(); + const internals = framework as unknown as { scriptToolCall(agent: string, tool: string, args: string, context: object): Promise<{ json: string }> }; + const answer = JSON.parse((await internals.scriptToolCall('prime', 'workspace--read', JSON.stringify({ path: 'files/a.txt' }), {})).json); + assert.deepEqual(answer, { + content: [{ type: 'text', text }], + structuredContent: { path: 'files/a.txt', totalLines: 1, fromLine: 1, toLine: 1, content: ' 1\tsay "hi"\\n\tthere' }, + isError: false, + }); + } finally { + await framework.stop(); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/workspace-text.test.ts b/test/workspace-text.test.ts index f1cb05dfa..f862e6729 100644 --- a/test/workspace-text.test.ts +++ b/test/workspace-text.test.ts @@ -3,7 +3,7 @@ // string's length as JSON, which a page's budget is counted in. import test from 'node:test'; import assert from 'node:assert/strict'; -import { fileText, hasBom, jsonLength, numberedLine, safeCut } from '../src/modules/workspace/text.js'; +import { fileText, hasBom, numberedLine, safeCut } from '../src/modules/workspace/text.js'; test('fileText gives the lines read shows, where each starts, and what the text is', () => { const lines = (text: string | Buffer) => fileText(Buffer.from(text)).lines; @@ -69,14 +69,3 @@ test('hasBom finds a UTF-8 byte order mark at the start alone', () => { } }); -test('jsonLength counts a string as JSON.stringify writes it', () => { - let mismatches = 0; - const check = (value: string) => { - if (jsonLength(value) !== JSON.stringify(value).length - 2) mismatches++; - }; - for (let c = 0; c <= 0xffff; c++) check(String.fromCharCode(c)); - // Escapes and surrogates beside each other: a pair, each half alone, and both in the wrong order. - const edges = [0x41, 0x22, 0x5c, 0x0a, 0x1f, 0x7f, 0x2028, 0xd800, 0xdbff, 0xdc00, 0xdfff, 0xd83d, 0xde00]; - for (const a of edges) for (const b of edges) for (const c of edges) check(String.fromCharCode(a, b, c)); - assert.equal(mismatches, 0); -}); diff --git a/test/workspace-three-way.test.ts b/test/workspace-three-way.test.ts index ac3d14e6a..276fb77ec 100644 --- a/test/workspace-three-way.test.ts +++ b/test/workspace-three-way.test.ts @@ -155,7 +155,7 @@ class Env { async function call(m: WorkspaceModule, name: string, input: Record): Promise { const res = await m.handleToolCall({ id: 't', name, input }); assert.equal(res.success, true, `${name} ${JSON.stringify(input)} failed: ${res.error}`); - return res.data; + return res.structured ?? res.data; } async function refused(m: WorkspaceModule, name: string, input: Record): Promise { @@ -1325,7 +1325,7 @@ describe("another mount's directory, to the outer mount's file tools and pushes" m.registerGeneratedTextFile('work/generated.md', () => 'generated', 'resident'); const generated = await m.handleToolCall({ id: 't', name: 'read', callerAgentName: 'resident', input: { path: 'work/generated.md' } }); assert.equal(generated.success, true, generated.error); - assert.match(JSON.stringify(generated.data), /generated/); + assert.match(JSON.stringify(generated.structured), /generated/); }); test('the path a refusal names works on a mount whose root is configured with a trailing separator', async (t) => { @@ -2697,21 +2697,33 @@ describe("a file's mode is kept with its bytes", () => { env.store.treeSet(TREE, 'a.sh', { blobHash: e.blobHash, size: e.size, mode: 0o100755 }); const restore = await refuseChmod(); let done: Record; + // The text the model reads of an edit's receipt. + let text = ''; + const edited = async (name: 'edit' | 'edit_many', input: Record) => { + const result = await m.handleToolCall({ id: 't', name, input }); + assert.equal(result.success, true, result.error); + text = (result.data as Array<{ text: string }>)[0]!.text; + return result.structured as Record; + }; try { done = tool === 'write' ? await call(m, 'write', { path: 'work/a.sh', content: 'echo a v2\n' }) : tool === 'edit' - ? await call(m, 'edit', { path: 'work/a.sh', oldString: 'echo a', newString: 'echo a v2' }) + ? await edited('edit', { path: 'work/a.sh', oldString: 'echo a', newString: 'echo a v2' }) : tool === 'edit_many' // edit_many gives it in the file's entry. - ? ((await call(m, 'edit_many', { edits: [{ path: 'work/a.sh', oldString: 'echo a', newString: 'echo a v2' }] })).files as Array>)[0]! + ? ((await edited('edit_many', { edits: [{ path: 'work/a.sh', oldString: 'echo a', newString: 'echo a v2' }] })).files as Array>)[0]! : ((await m.writeBinary('work/a.sh', Buffer.from('echo a v2\n'), 'text/x-sh')).data as Record); } finally { restore(); } assert.equal(env.readDisk('a.sh'), 'echo a v2\n'); assert.equal(modeOf(env, 'a.sh'), 0o644, "disk's mode kept"); - assert.equal(done.modeKept, "written, its mode left at 0644: the kernel refused the chmod, so the workspace's (0755) isn't given, and a sync takes disk's mode in"); + const kept = "written, its mode left at 0644: the kernel refused the chmod, so the workspace's (0755) isn't given, and a sync takes disk's mode in"; + assert.equal(done.modeKept, kept); + // The model reads it too: in edit's header, and on edit_many's line for the file. + if (tool === 'edit') assert.match(text, new RegExp(`\\nmodeKept: ${kept.replace(/[()]/g, '\\$&')}\\n--- changed ---\\n`)); + if (tool === 'edit_many') assert.match(text, new RegExp(`\\n--- files ---\\nwork/a\\.sh: size 10, replacements 1, edits 1; modeKept: ${kept.replace(/[()]/g, '\\$&')}\\n`)); assert.equal(env.events.filter((x) => (x as { type?: string }).type === 'workspace:materialize-failed').length, 0); }); } @@ -7330,7 +7342,7 @@ describe('on-agent-action', () => { env.store.switchBranch(main); const limited = await withReadHook(switching, () => m.handleToolCall({ id: 't', name: 'read', input: { path: 'work/a.txt' }, resultMaxChars: 1_000 })); assert.equal(limited.success, true, String(limited.error)); - assert.equal((limited.data as { note?: string }).note, data.note); + assert.equal((limited.structured as { note?: string }).note, data.note); }); test("a read of a conflict on a read-only mount names only the sync, since nothing writes the workspace's copy there", async (t) => { @@ -7517,7 +7529,7 @@ describe('on-agent-action', () => { writeFileSync(join(env.dir, 'a.txt'), 'A line\n'); const edit = await m.handleToolCall({ id: 't', name: 'edit', input: { path: 'work/a.txt', oldString: 'A line', newString: 'A edited' } }); assert.equal(edit.success, true, JSON.stringify(edit)); - assert.match(JSON.stringify(edit.data), /couldn't be checked against disk \(the mount root is not the directory disk last agreed with/); + assert.match(JSON.stringify(edit.structured), /couldn't be checked against disk \(the mount root is not the directory disk last agreed with/); }); test('a branch switched away during the root check and back during a read is decided on the branch enumerated', async (t) => { From c76fffaebbd0191acf87e4dc2617d3c394f1d5b8 Mon Sep 17 00:00:00 2001 From: Cedric-2714 Date: Fri, 23 Oct 2026 19:09:28 -0400 Subject: [PATCH 2/6] test(workspace): a receipt's region of many short lines keeps to its cap The line feed between each of a region's lines is part of what the receipt's text holds, and shownRegions counts it. Removing that count failed no test: the receipt tests' regions had few lines, so the reserve kept for the closing notes absorbed the shortfall. Three hundred one-character lines make the feeds a third of the region, and every cap from 2,000 to 3,500 is tried. Co-Authored-By: Claude Opus 5.5 --- test/workspace-edits.test.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/workspace-edits.test.ts b/test/workspace-edits.test.ts index 02147203c..514aec611 100644 --- a/test/workspace-edits.test.ts +++ b/test/workspace-edits.test.ts @@ -1574,6 +1574,24 @@ test('a receipt keeps aside only what its own call can add: one edit of a file o } }); +test("a receipt counts the line feed between each of its region's lines, so a region of many short lines keeps to its cap", () => { + // Three hundred lines of one character: each line feed is a third of a + // line's own length or more, so a region counted without them would show + // where its text passes the cap. + const bytes = Buffer.from('start\n'); + const planned = planFile(decodeForEdit(bytes)!, bytes, [{ index: 0, oldText: 'start', newText: Array.from({ length: 300 }, () => 'x').join('\n'), expected: 1 }]); + assert.ok('plan' in planned); + const files = [{ path: 'work/s.txt', plan: planned.plan, notes: [] }]; + let shown = 0; + for (let maxChars = 2_000; maxChars <= 3_500; maxChars++) { + const receipt = editReceipt('edit', files, 1, maxChars); + const length = receiptText(receipt).length; + if (length > maxChars) assert.fail(`${length} characters at a cap of ${maxChars}`); + if (!changedOf(receipt).startsWith('[not shown')) shown++; + } + assert.ok(shown > 0, 'the region shows at the larger caps'); +}); + test("what a receipt keeps aside doesn't grow with a line's length: twenty edits of a long line show their region with what twenty of a short one need", () => { // For the note naming the region of twenty edits on one line, a receipt // keeps aside twenty ranges that end by line 3, however long the line. Kept From b9a5b0016a9c60042db051690c219d618d558a64 Mon Sep 17 00:00:00 2001 From: Cedric-2714 Date: Fri, 23 Oct 2026 19:09:29 -0400 Subject: [PATCH 3/6] fix(workspace): answer the first internal review of the text form - grep's matched and context lines go through shownValue too: read and grep split lines at LF alone, so a line can keep a CR, U+2028 or U+2029, and in grep's text more sections follow it. - grep's context runs as grep writes them: matches whose windows meet or overlap are one run, each line once, a match marked `:` wherever it shows, and `--` only across lines left out. The test that pinned a `--` between touching windows is turned round. - shownValue quotes Unicode's mandatory line breaks, LF, VT, FF, CR, NEL, U+2028 and U+2029, escaping NEL after JSON as it does U+2028 and U+2029. - read-page.ts: contentCost's arm for no content couldn't run, so lengthAt adds the content's label and characters; the doc on a script's answer says its fields include the page's content, the limit counts the text alone, and when the two together can pass the script's limit, half of it, about 3.7 million characters at the default script memory; the moves-on comment says a pair is taken whole with room for one unit. - edit-text.ts: editReceipt's doc says what the base already did, that the line naming what a receipt leaves out comes after its counts and notes even where they leave it too little room. - Changelog: the script threshold scales with script memory; a page is never shorter unless its path or a note holds a line break; the spill advice is longer below a cap of about 12,300, not 2,300; connectome-host needs 0.11.0 or later, before which its viewers show such a file as empty; the line-break set and grep's quoted lines. - Tests: grep's quoted lines and its runs (overlap, gap, touching); grepText with line breaks in its pattern, note, skipped reasons and incomplete places, a label without a state and a skip's facts as it has them; a receipt's note and kept mode with line breaks; VT, FF and NEL in shownValue's unit test; the spill read-back page uses its room, so a halved limitChars fails; stale comments on JSON budgets. Co-Authored-By: Claude Opus 5.5 --- .../cedric-f55-workspace-text.breaking.md | 32 +++++--- src/modules/workspace/edit-text.ts | 6 +- src/modules/workspace/read-page.ts | 23 +++--- src/modules/workspace/result-text.ts | 39 +++++---- test/tool-result-cap-fit.test.ts | 6 +- test/tool-result-spill.test.ts | 3 + test/workspace-result-text.test.ts | 79 +++++++++++++++++-- test/workspace-text.test.ts | 3 +- 8 files changed, 143 insertions(+), 48 deletions(-) diff --git a/changelog.d/cedric-f55-workspace-text.breaking.md b/changelog.d/cedric-f55-workspace-text.breaking.md index 8192d0c03..195801b8c 100644 --- a/changelog.d/cedric-f55-workspace-text.breaking.md +++ b/changelog.d/cedric-f55-workspace-text.breaking.md @@ -5,9 +5,11 @@ of the result's fields, one `key: value` to a line (`path: …`, `totalLines: …`, `nextOffsetChars: null`), then labelled sections: a page's `--- content ---`, grep's matches by file and version (`N:` for a - match, `N-` for a line of its context), what grep skipped, and a receipt's - `--- files ---` and `--- changed ---`. A value holding a line break, such - as a path, is written as a JSON string. A control character in a file + match, `N-` for a line of its context, as grep writes them), what grep + skipped, and a receipt's `--- files ---` and `--- changed ---`. A value + outside a page's content and a receipt's changed lines that holds a line + break (LF, CR, VT, FF, NEL, U+2028 or U+2029), such as a path or a line + grep matched, is written as a JSON string. A control character in a file reaches the model as itself, where JSON wrote an escape such as `\u001b`. **Who needs to act:** @@ -17,7 +19,8 @@ - a host or module that reads these results' fields from `result.data`, through `executeToolCall` or `ctx.callTool`: read `result.structured`. connectome-host reads either since 0.11.0 - (anima-research/connectome-host#324). + (anima-research/connectome-host#324), and needs that version or later: + before it, its file viewers show such a file as empty. **What a page holds:** a page is held to the call's result limit as its text measures it, where it was measured as JSON, and so is a receipt. @@ -25,16 +28,19 @@ default 24,000 characters, a page of this repository's `src/modules/workspace/index.ts` now ends at line 475, where it ended at line 454. A spilled result's notice sizes the read-back page it suggests - the same way, so below a cap of about 2,300 characters it suggests a - longer one. + the same way, so below a cap of about 12,300 characters it suggests a + longer one than before. - A script's page is never shorter than before, since a page's text is - shorter than its JSON. A script's answer carries the fields beside the - text, which the limit doesn't count. Together they can pass what a - script takes only on a page of more than about 3.7 million characters - that holds control characters, which JSON writes as six bytes each where - no other character takes more than three. Such a page answers - `ResultTooLarge`, which cuts nothing, and the script reads a smaller - page. + shorter than its JSON, unless its path or a note holds a line break, + which the text quotes. A script's answer carries the result's fields, a + page's content among them, beside the text, and the limit counts the + text alone. A script's limit counts each character at the six bytes it + takes at most in the answer, and only a control character takes more + than three, so the two together can pass what the script takes only on + a page of more than half that limit (about 3.7 million characters at + the default 128 MiB of script memory) that holds control characters. + Such a page answers `ResultTooLarge`, which cuts nothing, and the script + reads a smaller page. **Unchanged:** the tools' descriptions and inputs; the words of their notes; their errors, which were text already; the results of `write`, diff --git a/src/modules/workspace/edit-text.ts b/src/modules/workspace/edit-text.ts index b4467017a..8f1598ebc 100644 --- a/src/modules/workspace/edit-text.ts +++ b/src/modules/workspace/edit-text.ts @@ -562,7 +562,11 @@ function manyHeadText(data: { edits: number; replacements: number; files: Listed * regions are named by their lines. edit_many lists its files while they * fit, a file whose notes don't fit without them, or without its edits' * numbers too, and counts the rest. edit's receipt, of one file, keeps its - * counts and notes and shows no region where those alone pass `maxChars`. + * counts and notes and shows no region where those leave no room for it. The + * line that names what a receipt leaves out comes after what it keeps, and + * where its counts and notes leave too little room for that line too, as + * they can with long notes or a limit far under the 1,000 a residence or an + * agent can set, the receipt passes `maxChars` by it. * The model reads the text; a script gets the receipt's fields beside it. */ export function editReceipt(tool: EditTool, files: StoredFile[], editCount: number, maxChars: number): ToolResult { diff --git a/src/modules/workspace/read-page.ts b/src/modules/workspace/read-page.ts index 2595ae318..0c07c14c3 100644 --- a/src/modules/workspace/read-page.ts +++ b/src/modules/workspace/read-page.ts @@ -8,10 +8,13 @@ import { type FileText, MAX_LINE_CHARS, numberedLine, safeCut } from './text.js' * limit doesn't reach it whole. A model's result spills into a file, cut, and * a script gets ResultTooLarge in its place. So a page stops before its text * would pass the limit, and its note says where to continue. A script's - * answer carries the page's fields beside its text, which the limit doesn't - * count but leaves room for, since it counts each character at the six bytes - * it takes at most in the script's answer: the two pass it only together, on - * a page of millions of characters, most of them control characters. + * answer carries the page's fields, its content among them, beside its text, + * and the limit counts the text alone. A script's limit counts each character + * at the six bytes it takes at most in the answer, and no character but a + * control character takes more than three, so the text and the fields + * together pass what the script takes only on a page of more than half that + * limit (about 3.7 million characters at the default 128 MiB of script + * memory) holding control characters. */ /** How many lines a read shows when it isn't given a limit. */ @@ -55,8 +58,6 @@ function pageText(page: Page): string { /** A page as read answers it: its text for the model, and its fields for a script. */ const pageAnswer = (page: Page): ToolResult => textAnswer(pageText(page), page); -/** What content adds to its page's text: its label's line with the line feed before it, and its own characters. */ -const contentCost = (chars: number): number => (chars > 0 ? sectionLead('content') + chars : 0); /** * Lines from `offset` (1-based), at most `limit` of them, each numbered and a @@ -119,9 +120,9 @@ export function linePage(path: string, file: FileText, offset: number, limit: nu return joined([to === last ? endNote : budgetNote(to, rest), ...notes]); }; // A page's text's length: its fields' and note's, measured with no content, - // and its content's, under the content's label. + // and its content's, under the content's label. A page here has a line. const lengthAt = (to: number, contentChars: number, note: string | undefined) => - pageText(page(to, '', note)).length + contentCost(contentChars); + pageText(page(to, '', note)).length + sectionLead('content') + contentChars; // The page is the longest that fits. Its length isn't monotone in its // lines: the page that ends the file carries no stop note, the one at the // line limit a shorter one than a stop, and one whose last line is cut a @@ -213,8 +214,10 @@ export function characterPage(path: string, file: FileText, offsetChars: number, let to = Math.min(end, start + Math.max(0, maxChars - overhead)); if (splitsPair(text, to)) to--; // A page always moves on, by one character at least, though a limit too - // small for the page's own fields then passes it. Where that reaches the - // page's own end, it holds all it was asked for, so it names no stop. + // small for the page's own fields and its first character then passes it: + // with room for one code unit, a surrogate pair is still taken whole. Where + // that reaches the page's own end, it holds all it was asked for, so it + // names no stop. if (to === start) to = Math.min(end, start + step(start)); return pageAnswer(page(to, joined([to < end ? stoppedNote(to, end - to) : null, ...notes]))); } diff --git a/src/modules/workspace/result-text.ts b/src/modules/workspace/result-text.ts index 71a1248d5..967f762ae 100644 --- a/src/modules/workspace/result-text.ts +++ b/src/modules/workspace/result-text.ts @@ -9,19 +9,21 @@ import type { ToolResult } from '../../types/events.js'; * receipt is held to the call's result limit as this text measures it. */ -const LINE_BREAK = /[\n\r\u2028\u2029]/; +/** The characters Unicode says end a line (UAX #14's mandatory breaks): LF, VT, FF, CR, NEL, U+2028 and U+2029. */ +const LINE_BREAK = /[\n\v\f\r\u0085\u2028\u2029]/; /** * A value as a header line, a section's label or a listed line writes it: as * it is, or, where it holds a line break, as a JSON string. A path can hold - * one, and so can a note naming it; written as it is, it would start a line - * that reads as one of the answer's own, and the quoted form gives back - * exactly what the value holds. JSON leaves U+2028 and U+2029 as they are, - * so they're escaped too. + * one, and so can a note naming it or a line of a file grep matched; written + * as it is, it would start a line that reads as one of the answer's own, and + * the quoted form gives back exactly what the value holds. JSON leaves NEL, + * U+2028 and U+2029 as they are, so they're escaped too. */ export function shownValue(value: string | number | boolean | null): string { if (typeof value !== 'string') return String(value); - return LINE_BREAK.test(value) ? JSON.stringify(value).replace(/\u2028/g, '\\u2028').replace(/\u2029/g, '\\u2029') : value; + if (!LINE_BREAK.test(value)) return value; + return JSON.stringify(value).replace(/[\u0085\u2028\u2029]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`); } /** A header line: a field's key and its value as shownValue writes it. */ @@ -63,11 +65,14 @@ export interface GrepAnswer { /** * grep's text: its pattern, match count and note, then a section for each * version of a file it matched, labelled with the file, the version, the - * file's state and its line ends where it has them, then what it skipped and - * where its search was incomplete. A match's line reads `N:text`, as grep - * writes one, and where the call asked for context, each line around it reads - * `N-text`, with `--` between one match's lines and the next's. Each match's - * context starts `contextBefore` lines before it, or at the file's first line. + * file's state where grep gives one and its line ends where it has them, then + * what it skipped and where its search was incomplete. As grep writes them, a + * match's line reads `N:text`, and where the call asked for context, each + * line around it reads `N-text`: matches whose context meets or overlaps are + * one run of lines, each line once, and `--` stands between runs with lines + * left out between them. Each match's context starts `contextBefore` lines + * before it, or at the file's first line. A line's text is written through + * shownValue, since more of the answer follows it. */ export function grepText(answer: GrepAnswer, contextBefore: number): string { const header = [fieldLine('pattern', answer.pattern), fieldLine('totalMatches', answer.totalMatches)]; @@ -77,16 +82,22 @@ export function grepText(answer: GrepAnswer, contextBefore: number): string { + (result.state !== undefined ? ` (${result.state})` : '') + (result.lineEndings !== undefined ? `; lineEndings: ${result.lineEndings}` : ''); const lines: string[] = []; + const matched = new Set(result.matches.map((match) => match.line)); + // The last line written: matches come in line order, so a window that + // meets or overlaps the run before it goes on from there. + let last = 0; for (const match of result.matches) { if (match.context === undefined) { - lines.push(`${String(match.line).padStart(6)}:${match.text}`); + lines.push(`${String(match.line).padStart(6)}:${shownValue(match.text)}`); continue; } - if (lines.length > 0) lines.push('--'); const first = Math.max(1, match.line - contextBefore); match.context.forEach((text, k) => { const n = first + k; - lines.push(`${String(n).padStart(6)}${n === match.line ? ':' : '-'}${text}`); + if (n <= last) return; + if (last > 0 && n > last + 1) lines.push('--'); + lines.push(`${String(n).padStart(6)}${matched.has(n) ? ':' : '-'}${shownValue(text)}`); + last = n; }); } return sectionText(label, lines.join('\n')); diff --git a/test/tool-result-cap-fit.test.ts b/test/tool-result-cap-fit.test.ts index 9077cce38..a7974bb31 100644 --- a/test/tool-result-cap-fit.test.ts +++ b/test/tool-result-cap-fit.test.ts @@ -137,9 +137,9 @@ const countWrites = (writes: string[]) => (framework: AgentFramework) => { }; /** A page the notice advises comes back within the inline cap of 4,000 when - * read alone in its own turn (each read-back code unit can take six - * characters, beside the page's metadata), and is sized to that cap, not - * to the `share` of the turn that cut the result. */ + * read alone in its own turn (each read-back code unit takes one character + * of the page's text, beside its fields), and is sized to that cap, not to + * the `share` of the turn that cut the result. */ const assertPageForLaterRead = (limit: number, quotedPath: string, share: number) => { const overhead = (JSON.parse(quotedPath) as string).length + 256; assert.ok(limit + overhead <= 4000, `a page of ${limit} comes back within the cap`); diff --git a/test/tool-result-spill.test.ts b/test/tool-result-spill.test.ts index fdddd52f0..03763fe1e 100644 --- a/test/tool-result-spill.test.ts +++ b/test/tool-result-spill.test.ts @@ -358,6 +358,9 @@ describe('tool-result spill completion (issue #89)', () => { const outcome = storage.spilled.get(`page-${pages}`)!; assert.strictEqual(outcome.filePath, null, 'read-back must not spill again'); assert.ok(outcome.text.length <= cap); + // And it uses its room: the advised page is sized to the cap, up to + // the 2,000 characters a page is advised at the most. + if (page.nextOffsetChars !== null) assert.ok(outcome.text.length > Math.min(cap / 2, 2_000), `a page of ${outcome.text.length} at a cap of ${cap}`); assert.strictEqual(outcome.text, (result.data as Array<{ text: string }>)[0]!.text, 'the page is stored as its text'); assert.ok(outcome.text.endsWith(`\n--- content ---\n${page.content}`), 'which ends with its content'); recovered += page.content; diff --git a/test/workspace-result-text.test.ts b/test/workspace-result-text.test.ts index c8ad9e6ac..d14427871 100644 --- a/test/workspace-result-text.test.ts +++ b/test/workspace-result-text.test.ts @@ -14,7 +14,9 @@ import { dirname, join } from 'node:path'; import { JsStore } from '@animalabs/chronicle'; import { AgentFramework } from '../src/index.js'; import { WorkspaceModule } from '../src/modules/workspace/index.js'; -import { shownValue } from '../src/modules/workspace/result-text.js'; +import { type GrepAnswer, grepText, shownValue } from '../src/modules/workspace/result-text.js'; +import { decodeForEdit, planFile } from '../src/modules/workspace/edit-plan.js'; +import { editReceipt } from '../src/modules/workspace/edit-text.js'; import type { EventResponse, Module, ModuleContext } from '../src/types/module.js'; import type { ProcessEvent, ToolResult } from '../src/types/events.js'; import { MockMembrane, createMockResponse } from './helpers/mock-membrane.js'; @@ -187,14 +189,14 @@ test("grep answers each version of a file it matched as a section, a match as N: assert.deepEqual(fields<{ results: Array<{ file: string; version: string }> }>(found).results.map((r) => `${r.file} ${r.version}`), ['work/a.txt workspace', 'work/b.txt workspace', 'work/b.txt conflicting-disk', 'work/c.txt workspace']); - // Context starts at the file's first line at the earliest, and `--` parts one match's lines from the next's. + // Context starts at the file's first line at the earliest, and a window + // that begins where the run before it ends goes on in the same run. const context = await call('grep', { pattern: 'needle', path: 'work/a.txt', contextBefore: 1 }, 24_000); assert.equal(textOf(context), [ 'pattern: needle', 'totalMatches: 2', '--- work/a.txt: workspace (synced) ---', ' 1:needle one', - '--', ' 2-hay', ' 3:needle again', ].join('\n')); @@ -268,9 +270,9 @@ test("an edit's receipt is held to the call's limit as its text measures it", as }); test('a value holding a line break is written as a JSON string, so it stays on its line and gives back the path', async () => { - for (const value of ['a\nb', 'a\rb', 'a\u2028b', 'a\u2029b', '\n']) { + for (const value of ['a\nb', 'a\rb', 'a\u2028b', 'a\u2029b', 'a\vb', 'a\fb', 'a\u0085b', '\n']) { const shown = shownValue(value); - assert.ok(!/[\n\r\u2028\u2029]/.test(shown), `${JSON.stringify(value)} is shown on one line`); + assert.ok(!/[\n\v\f\r\u0085\u2028\u2029]/.test(shown), `${JSON.stringify(value)} is shown on one line`); assert.equal(JSON.parse(shown), value); } for (const value of ['plain', 'tab\tand \u001b escape', '"quoted"', '']) assert.equal(shownValue(value), value); @@ -349,3 +351,70 @@ test("through the framework, a model's read reaches it and its history as the pa rmSync(root, { recursive: true, force: true }); } }); + +test("grep's matched and context lines that hold a line break are quoted, and matches whose context meets are one run of lines", async () => { + await withWorkspace(async ({ call, disk }) => { + disk('lines.txt', 'one\ntwo \r needle three\nfour\nneedle five\u2028--- skipped ---\nsix\n'); + const quoted = await call('grep', { pattern: 'needle', path: 'work/lines.txt', contextBefore: 1, contextAfter: 1 }, 24_000); + // The two windows, lines 1 to 3 and 3 to 5, overlap: one run, each line once. + assert.equal(textOf(quoted), [ + 'pattern: needle', + 'totalMatches: 2', + '--- work/lines.txt: workspace (synced) ---', + ' 1-one', + ' 2:"two \\r needle three"', + ' 3-four', + ' 4:"needle five\\u2028--- skipped ---"', + ' 5-six', + ].join('\n')); + disk('gap.txt', `${['a', 'needle b', 'c', 'd', 'e', 'f', 'g', 'needle h', 'i'].join('\n')}\n`); + // Lines left out between two runs: `--` between them. + assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/gap.txt', contextBefore: 1, contextAfter: 1 }, 24_000)), + /\n {5}1-a\n {5}2:needle b\n {5}3-c\n--\n {5}7-g\n {5}8:needle h\n {5}9-i$/); + disk('touch.txt', 'x\nneedle y\nz\nneedle w\n'); + // A window that ends where the next begins: one run, a match inside another's window marked as a match. + assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/touch.txt', contextAfter: 2 }, 24_000)), + /\n {5}2:needle y\n {5}3-z\n {5}4:needle w$/); + assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/touch.txt', contextAfter: 1 }, 24_000)), + /\n {5}2:needle y\n {5}3-z\n {5}4:needle w$/); + }); +}); + +test("grep's text quotes a pattern, a note, a skipped file's reason and an incomplete place that hold a line break, and writes a label without a state and a skip's facts as it has them", () => { + const answer: GrepAnswer = { + pattern: 'a\nb', + results: [{ file: 'work/d.txt', version: 'disk', matches: [{ line: 1, text: 'x' }] }], + totalMatches: 1, + note: 'see "work/q\nr"', + skipped: [ + { file: 'work/s', reason: 'gone\nwhere' }, + { file: 'work/t', reason: 'big', size: 9 }, + { file: 'work/u.png', reason: 'img', mimeType: 'image/png' }, + ], + incomplete: [{ path: 'work/i\nj', reason: 'nested\rmount' }], + }; + assert.equal(grepText(answer, 0), [ + 'pattern: "a\\nb"', + 'totalMatches: 1', + 'note: "see \\"work/q\\nr\\""', + '--- work/d.txt: disk ---', + ' 1:x', + '--- skipped ---', + 'work/s: "gone\\nwhere"', + 'work/t: big (9 bytes)', + 'work/u.png: img (image/png)', + '--- incomplete ---', + '"work/i\\nj": "nested\\rmount"', + ].join('\n')); +}); + +test("a receipt quotes a note and a kept mode that hold a line break, in edit's header and on edit_many's line for the file", () => { + const bytes = Buffer.from('old\n'); + const planned = planFile(decodeForEdit(bytes)!, bytes, [{ index: 0, oldText: 'old', newText: 'new', expected: 1 }]); + assert.ok('plan' in planned); + const file = { path: 'work/a.txt', plan: planned.plan, notes: ['note\nwith a break'], modeKept: 'kept\u2028here' }; + assert.match(textOf(editReceipt('edit', [file], 1, 24_000)), + /\nnote: "note\\nwith a break"\nmodeKept: "kept\\u2028here"\n--- changed ---\n/); + assert.match(textOf(editReceipt('edit_many', [file, { path: 'work/b.txt', plan: planned.plan, notes: [] }], 2, 24_000)), + /\n--- files ---\nwork\/a\.txt: size 4, replacements 1, edits 1; modeKept: "kept\\u2028here"\. "note\\nwith a break"\nwork\/b\.txt: size 4, replacements 1, edits 1\n/); +}); diff --git a/test/workspace-text.test.ts b/test/workspace-text.test.ts index f862e6729..83bda24b5 100644 --- a/test/workspace-text.test.ts +++ b/test/workspace-text.test.ts @@ -1,6 +1,5 @@ // The text helpers the workspace's read, grep and edit share: a file's lines as -// read shows them, numbered lines with a long one cut, a BOM's check, and a -// string's length as JSON, which a page's budget is counted in. +// read shows them, numbered lines with a long one cut, and a BOM's check. import test from 'node:test'; import assert from 'node:assert/strict'; import { fileText, hasBom, numberedLine, safeCut } from '../src/modules/workspace/text.js'; From a4e547c15907c3265e5b5b57c544d77a189b4d57 Mon Sep 17 00:00:00 2001 From: Cedric-2714 Date: Fri, 23 Oct 2026 19:09:29 -0400 Subject: [PATCH 4/6] test(workspace): grep's matched lines without context are quoted too Taking shownValue off a match written without context failed no test: the quoting test asked for context, so its matches went through the context path. The same file grepped without context now shows both matched lines quoted. Co-Authored-By: Claude Opus 5.5 --- test/workspace-result-text.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/workspace-result-text.test.ts b/test/workspace-result-text.test.ts index d14427871..b955c5e5e 100644 --- a/test/workspace-result-text.test.ts +++ b/test/workspace-result-text.test.ts @@ -367,6 +367,9 @@ test("grep's matched and context lines that hold a line break are quoted, and ma ' 4:"needle five\\u2028--- skipped ---"', ' 5-six', ].join('\n')); + // Without context, the matched lines are quoted the same way. + assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/lines.txt' }, 24_000)), + /\n {5}2:"two \\r needle three"\n {5}4:"needle five\\u2028--- skipped ---"$/); disk('gap.txt', `${['a', 'needle b', 'c', 'd', 'e', 'f', 'g', 'needle h', 'i'].join('\n')}\n`); // Lines left out between two runs: `--` between them. assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/gap.txt', contextBefore: 1, contextAfter: 1 }, 24_000)), From 621f312d77002da6ff064dc080c1a8808d672d03 Mon Sep 17 00:00:00 2001 From: Cedric-2714 Date: Fri, 23 Oct 2026 19:09:29 -0400 Subject: [PATCH 5/6] test(workspace): answer the second internal review of the text form - grep's touching-window assertions compare the whole text, so a `--` before a section's first run fails them. - A new test greps three files with context: each file's run starts afresh, and a context-only line holding a lone CR is quoted. - shownValue's test asserts the exact quoted form of a value holding NEL, two U+2028s, a U+2029 and a VT, pinning each escape and its four digits. - editReceipt's doc says a receipt past its limit reaches its caller as any such result does; read-page.ts loses a double blank line. Co-Authored-By: Claude Opus 5.5 --- src/modules/workspace/edit-text.ts | 6 ++++-- src/modules/workspace/read-page.ts | 1 - test/workspace-result-text.test.ts | 34 +++++++++++++++++++++++++----- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/modules/workspace/edit-text.ts b/src/modules/workspace/edit-text.ts index 8f1598ebc..42162d2b3 100644 --- a/src/modules/workspace/edit-text.ts +++ b/src/modules/workspace/edit-text.ts @@ -566,8 +566,10 @@ function manyHeadText(data: { edits: number; replacements: number; files: Listed * line that names what a receipt leaves out comes after what it keeps, and * where its counts and notes leave too little room for that line too, as * they can with long notes or a limit far under the 1,000 a residence or an - * agent can set, the receipt passes `maxChars` by it. - * The model reads the text; a script gets the receipt's fields beside it. + * agent can set, the receipt passes `maxChars` by it, and then reaches its + * caller as any result past its limit does: a model's spills, and a script's + * call answers ResultTooLarge. Otherwise the model reads the text, and a + * script gets the receipt's fields beside it. */ export function editReceipt(tool: EditTool, files: StoredFile[], editCount: number, maxChars: number): ToolResult { const reserve = receiptReserve(tool, files, maxChars); diff --git a/src/modules/workspace/read-page.ts b/src/modules/workspace/read-page.ts index 0c07c14c3..3dbb3b706 100644 --- a/src/modules/workspace/read-page.ts +++ b/src/modules/workspace/read-page.ts @@ -58,7 +58,6 @@ function pageText(page: Page): string { /** A page as read answers it: its text for the model, and its fields for a script. */ const pageAnswer = (page: Page): ToolResult => textAnswer(pageText(page), page); - /** * Lines from `offset` (1-based), at most `limit` of them, each numbered and a * long one cut with the offsetChars and limitChars that read the rest. The diff --git a/test/workspace-result-text.test.ts b/test/workspace-result-text.test.ts index b955c5e5e..6ff694062 100644 --- a/test/workspace-result-text.test.ts +++ b/test/workspace-result-text.test.ts @@ -276,6 +276,8 @@ test('a value holding a line break is written as a JSON string, so it stays on i assert.equal(JSON.parse(shown), value); } for (const value of ['plain', 'tab\tand \u001b escape', '"quoted"', '']) assert.equal(shownValue(value), value); + // Every break is escaped, each as JSON's four-digit escape. + assert.equal(shownValue('a\u0085b\u2028c\u2029d\u2028e\vf'), '"a\\u0085b\\u2028c\\u2029d\\u2028e\\u000bf"'); assert.equal(shownValue(null), 'null'); assert.equal(shownValue(12), '12'); @@ -375,11 +377,12 @@ test("grep's matched and context lines that hold a line break are quoted, and ma assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/gap.txt', contextBefore: 1, contextAfter: 1 }, 24_000)), /\n {5}1-a\n {5}2:needle b\n {5}3-c\n--\n {5}7-g\n {5}8:needle h\n {5}9-i$/); disk('touch.txt', 'x\nneedle y\nz\nneedle w\n'); - // A window that ends where the next begins: one run, a match inside another's window marked as a match. - assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/touch.txt', contextAfter: 2 }, 24_000)), - /\n {5}2:needle y\n {5}3-z\n {5}4:needle w$/); - assert.match(textOf(await call('grep', { pattern: 'needle', path: 'work/touch.txt', contextAfter: 1 }, 24_000)), - /\n {5}2:needle y\n {5}3-z\n {5}4:needle w$/); + // A window that takes in the next match, and one that ends where the + // next begins: one run each time, a match inside another's window + // marked as a match, and no `--` before a section's first run. + const touching = ['pattern: needle', 'totalMatches: 2', '--- work/touch.txt: workspace (synced) ---', ' 2:needle y', ' 3-z', ' 4:needle w'].join('\n'); + assert.equal(textOf(await call('grep', { pattern: 'needle', path: 'work/touch.txt', contextAfter: 2 }, 24_000)), touching); + assert.equal(textOf(await call('grep', { pattern: 'needle', path: 'work/touch.txt', contextAfter: 1 }, 24_000)), touching); }); }); @@ -421,3 +424,24 @@ test("a receipt quotes a note and a kept mode that hold a line break, in edit's assert.match(textOf(editReceipt('edit_many', [file, { path: 'work/b.txt', plan: planned.plan, notes: [] }], 2, 24_000)), /\n--- files ---\nwork\/a\.txt: size 4, replacements 1, edits 1; modeKept: "kept\\u2028here"\. "note\\nwith a break"\nwork\/b\.txt: size 4, replacements 1, edits 1\n/); }); + +test("grep's runs start afresh in each file's section, and a context line holding a line break is quoted", async () => { + await withWorkspace(async ({ call, disk }) => { + disk('two/a.txt', 'x\nneedle\n'); + disk('two/b.txt', 'needle\ny\n'); + disk('two/c.txt', 'pre\rcontext\nneedle\n'); + assert.equal(textOf(await call('grep', { pattern: 'needle', path: 'work/two', contextBefore: 1, contextAfter: 1 }, 24_000)), [ + 'pattern: needle', + 'totalMatches: 3', + '--- work/two/a.txt: workspace (synced) ---', + ' 1-x', + ' 2:needle', + '--- work/two/b.txt: workspace (synced) ---', + ' 1:needle', + ' 2-y', + '--- work/two/c.txt: workspace (synced) ---', + ' 1-"pre\\rcontext"', + ' 2:needle', + ].join('\n')); + }); +}); From c3218abeee2b87be5c5d175b934260f3cd5270d2 Mon Sep 17 00:00:00 2001 From: Cedric-2714 Date: Fri, 23 Oct 2026 19:21:25 -0400 Subject: [PATCH 6/6] fix(workspace): quote a shown value that starts with a quote A value the text shows as it is could itself be a JSON string: a line of a file reading "a\nb" (a quote, a, a backslash, n, b, a quote) read exactly as the quoted form of a, a line break and b. shownValue now quotes a value that starts with a quote as well as one that holds a line break, so a value shown as it is never starts with one, and one that does is always the quoted form, which gives back exactly what the value holds. The changelog says so. Co-Authored-By: Claude Opus 5.5 --- .../cedric-f55-workspace-text.breaking.md | 8 ++++---- src/modules/workspace/result-text.ts | 16 ++++++++++------ test/workspace-result-text.test.ts | 9 ++++++++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/changelog.d/cedric-f55-workspace-text.breaking.md b/changelog.d/cedric-f55-workspace-text.breaking.md index 195801b8c..b2efa7aef 100644 --- a/changelog.d/cedric-f55-workspace-text.breaking.md +++ b/changelog.d/cedric-f55-workspace-text.breaking.md @@ -8,8 +8,9 @@ match, `N-` for a line of its context, as grep writes them), what grep skipped, and a receipt's `--- files ---` and `--- changed ---`. A value outside a page's content and a receipt's changed lines that holds a line - break (LF, CR, VT, FF, NEL, U+2028 or U+2029), such as a path or a line - grep matched, is written as a JSON string. A control character in a file + break (LF, CR, VT, FF, NEL, U+2028 or U+2029) or starts with `"`, such as + a path or a line grep matched, is written as a JSON string, so a value + shown as it is never starts with `"`. A control character in a file reaches the model as itself, where JSON wrote an escape such as `\u001b`. **Who needs to act:** @@ -31,8 +32,7 @@ the same way, so below a cap of about 12,300 characters it suggests a longer one than before. - A script's page is never shorter than before, since a page's text is - shorter than its JSON, unless its path or a note holds a line break, - which the text quotes. A script's answer carries the result's fields, a + shorter than its JSON, unless the text quotes its path or a note. A script's answer carries the result's fields, a page's content among them, beside the text, and the limit counts the text alone. A script's limit counts each character at the six bytes it takes at most in the answer, and only a control character takes more diff --git a/src/modules/workspace/result-text.ts b/src/modules/workspace/result-text.ts index 967f762ae..f41ebdf04 100644 --- a/src/modules/workspace/result-text.ts +++ b/src/modules/workspace/result-text.ts @@ -14,15 +14,19 @@ const LINE_BREAK = /[\n\v\f\r\u0085\u2028\u2029]/; /** * A value as a header line, a section's label or a listed line writes it: as - * it is, or, where it holds a line break, as a JSON string. A path can hold - * one, and so can a note naming it or a line of a file grep matched; written - * as it is, it would start a line that reads as one of the answer's own, and - * the quoted form gives back exactly what the value holds. JSON leaves NEL, - * U+2028 and U+2029 as they are, so they're escaped too. + * it is, or, where it holds a line break or starts with a quote, as a JSON + * string. A path can hold a line break, and so can a note naming it or a line + * of a file grep matched; written as it is, it would start a line that reads + * as one of the answer's own. A value written as it is never starts with a + * quote, so one that does is always the quoted form, which gives back exactly + * what the value holds: without that, a value that is itself a JSON string, + * such as a line of a file reading `"a\nb"`, would read as the quoted form of + * another. JSON leaves NEL, U+2028 and U+2029 as they are, so they're escaped + * too. */ export function shownValue(value: string | number | boolean | null): string { if (typeof value !== 'string') return String(value); - if (!LINE_BREAK.test(value)) return value; + if (!value.startsWith('"') && !LINE_BREAK.test(value)) return value; return JSON.stringify(value).replace(/[\u0085\u2028\u2029]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, '0')}`); } diff --git a/test/workspace-result-text.test.ts b/test/workspace-result-text.test.ts index 6ff694062..5c2bd7945 100644 --- a/test/workspace-result-text.test.ts +++ b/test/workspace-result-text.test.ts @@ -275,7 +275,14 @@ test('a value holding a line break is written as a JSON string, so it stays on i assert.ok(!/[\n\v\f\r\u0085\u2028\u2029]/.test(shown), `${JSON.stringify(value)} is shown on one line`); assert.equal(JSON.parse(shown), value); } - for (const value of ['plain', 'tab\tand \u001b escape', '"quoted"', '']) assert.equal(shownValue(value), value); + for (const value of ['plain', 'tab\tand \u001b escape', 'a "quoted" word', '']) assert.equal(shownValue(value), value); + // A value that starts with a quote is quoted, so it can't read as the + // quoted form of another: `"a\nb"` as written in a file, and a line break + // between a and b, are shown apart. + assert.equal(shownValue('"quoted"'), '"\\"quoted\\""'); + assert.equal(shownValue('"a\\nb"'), '"\\"a\\\\nb\\""'); + assert.equal(shownValue('a\nb'), '"a\\nb"'); + for (const value of ['"quoted"', '"a\\nb"', 'a\nb', '"']) assert.equal(JSON.parse(shownValue(value)), value); // Every break is escaped, each as JSON's four-digit escape. assert.equal(shownValue('a\u0085b\u2028c\u2029d\u2028e\vf'), '"a\\u0085b\\u2028c\\u2029d\\u2028e\\u000bf"'); assert.equal(shownValue(null), 'null');