diff --git a/bench/probes/README.md b/bench/probes/README.md index 0f5b3bdd..dfe72cb9 100644 --- a/bench/probes/README.md +++ b/bench/probes/README.md @@ -62,3 +62,21 @@ rendered the SDK declarations seven times; it now takes four at depth three and renders them four times, and the final assembly carries `harness:identity`, `tools:ptc-only`, `dsh-ops:repository-tooling:mode-ptc` and `tools:sdk` with `run_code` as its only tool. + +## #982 — one session scope, and examples that are not copyable twice + +`probe-scope-guidance.mjs` runs every fenced `ts` example of one guidance file twice +inside one persistent REPL and reports which ones die on a redeclared top-level name. +The harness is Node's REPL with a retained context, the shape PTC evaluates in, so the +second run answers the only question the text controls: is a trajectory safe to copy +twice. + +```console +node bench/probes/probe-scope-guidance.mjs prompts/batching-guidance.md +``` + +On the revision-8 text it reports five examples and four that are not copyable twice +(`load`, `dirs`, `hits`, `rows`). The fifth, the verified-edit trajectory, uses +top-level `await`, which this plain REPL harness cannot evaluate, so it is neither +counted nor exonerated. The number is descriptive — any example with a top-level +`const` collides on its second run — and is never a gate. diff --git a/bench/probes/probe-scope-guidance.mjs b/bench/probes/probe-scope-guidance.mjs new file mode 100644 index 00000000..332eb66d --- /dev/null +++ b/bench/probes/probe-scope-guidance.mjs @@ -0,0 +1,106 @@ +/** + * Runs every fenced `ts` example of one guidance file TWICE inside one persistent + * REPL, the way a model copies a trajectory and later re-uses its shape. + * + * PTC evaluates each program in a retained REPL, so a top-level declaration outlives + * the program: the second run of the same example is a compile-time redeclaration + * unless the body is scoped. This probe answers the only question the guidance text + * controls — is a trajectory safe to copy twice — without a model and without tokens. + * + * Usage: node bench/probes/probe-scope-guidance.mjs [--json] + * + * The `return` normalization is deliberate: `repl` rejects a top-level return, and + * leaving it in would make the example fail to compile for an unrelated reason and + * hide the redeclaration this probe measures. Top-level `await` is not normalized: + * this plain REPL cannot evaluate it, so that example is reported as a first-run + * failure and neither counted nor exonerated. + * + * Baseline on the revision-8 file: 5 examples, 4 not copyable twice. + */ +import { readFileSync } from 'node:fs' +import { PassThrough, Writable } from 'node:stream' +import { start } from 'node:repl' + +/** Minimal typed surface every example calls; values never matter, only scoping does. */ +const tools = { + ops_load_sdk: async () => ({ status: 'completed', data: { tools: ['glob'] } }), + ops_glob: async () => ({ status: 'completed', data: { items: [{ path: 'src/a.ts' }] } }), + ops_grep: async () => ({ status: 'completed', data: { items: [{ path: 'src/a.ts' }] } }), + ops_read_files: async () => ({ status: 'completed', data: { items: [{ items: [], lineCount: 1, complete: true, nextOffset: 0 }] } }), + ops_inspect_local_file: async () => ({ status: 'completed', data: { items: [{ lineCount: 1, text: '' }] } }), + ops_stat: async () => ({ status: 'completed', data: { revision: 'rev-1' } }), + ops_edit_file: async () => ({ status: 'completed', data: { changed: true } }), +} + +/** Every fenced `ts` block that carries a body (the bare plan-line template has none). */ +function examples(markdown) { + return [...markdown.matchAll(/```ts\n([\s\S]*?)```/g)] + .map(match => match[1].replace(/\n+$/u, '')) + .filter(block => !block.includes('')) +} + +/** Give the trailing `return X` a name so the repl accepts it; scoping is untouched. */ +function normalizeReturn(block, index) { + // A name per example: a shared one would collide across examples and report a + // redeclaration the guidance never caused. + return block.replace(/^return /mu, `const __out${index} = `) +} + +/** Evaluate one program in the shared REPL; syntax errors print instead of calling back. */ +function evaluate(repl, code) { + return new Promise(resolve => { + const printed = [] + repl._probeOut.length = 0 + let settled = false + const finish = value => { if (!settled) { settled = true; resolve(value) } } + const timer = setTimeout(() => finish({ error: null, text: repl._probeOut.join('').trim() }), 400) + repl.eval(code + '\n', repl.context, 'ptc-repl', (error, value) => { + clearTimeout(timer) + finish({ error: error ? error.message : null, text: printed.join(''), value }) + }) + }) +} + +const path = process.argv[2] +if (!path) { + console.error('usage: node bench/probes/probe-scope-guidance.mjs [--json]') + process.exit(2) +} +const captured = [] +const input = new PassThrough() +const output = new Writable({ write(chunk, _encoding, callback) { captured.push(chunk.toString()); callback() } }) +const repl = start({ input, output, terminal: false, prompt: '', useGlobal: false, ignoreUndefined: true }) +Object.defineProperty(repl.context, 'tools', { configurable: true, enumerable: true, writable: true, value: tools }) +Object.defineProperty(repl, '_probeOut', { configurable: true, enumerable: false, writable: true, value: captured }) + +const blocks = examples(readFileSync(path, 'utf8')) +const rows = [] +for (const [index, block] of blocks.entries()) { + const code = normalizeReturn(block, index + 1) + const first = await evaluate(repl, code) + const second = await evaluate(repl, code) + const redeclared = /already been declared/u.test(second.text) + rows.push({ + example: index + 1, + plan: (block.split('\n')[0] || '').slice(0, 58), + first: first.error ?? (/Uncaught/u.test(first.text) ? first.text.split('\n')[0] : 'ok'), + second: second.error ?? (/Uncaught/u.test(second.text) ? second.text.split('\n')[0] : 'ok'), + redeclared, + }) +} +repl.close() +input.destroy() +output.destroy() + +const failed = rows.filter(row => row.redeclared) +if (process.argv.includes('--json')) { + console.log(JSON.stringify({ file: path, examples: rows.length, redeclared: failed.length, rows }, null, 2)) +} else { + console.log(`${path}: ${rows.length} example(s), ${failed.length} not copyable twice`) + for (const row of rows) { + console.log(` ${row.redeclared ? 'REDECLARE' : 'ok '} #${row.example} ${row.plan}`) + if (row.redeclared) console.log(` second run: ${row.second}`) + else if (row.first !== 'ok') console.log(` first run: ${row.first}`) + } +} +process.exitCode = failed.length > 0 ? 1 : 0 diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 67d397ff..cd8dd1b8 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,26 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- The batch guidance states the scope every PTC program shares: the runtime evaluates each program in a REPL + whose context is retained, so a program's top-level `const`, `let`, `class` and `function` outlive it, and a + second declaration of that name is a compile-time SyntaxError that discards the whole program before its + first line runs, tool calls included. A failed program may already have run calls and keeps its names, so a + retry checks the effects and renames first. One short section carries the three rules, and the mistake table + gains the two matching rows; the 0.5.6 sandbox section, examples, steps and the four trajectories are + unchanged, and a loader-less surface keeps the rule. Measured on one 127-program session, 11 of the 17 + failed programs were this exact error, two of them retries of a program that had already failed; the section + adds 803 B to the lesson (8,169 to 8,972 B) and `bench/probes/probe-scope-guidance.mjs` reports the shipped + revision-9 text as five examples, four of them not copyable twice (`load`, `dirs`, `hits`, `rows`). Revision 9; + no behaviour changes. +- 批量引导写明每个 PTC 程序共享的那个作用域:运行时在保留上下文的 REPL 里求值,程序的顶层 + `const`/`let`/`class`/`function` 会比程序活得久,同名再声明是编译期 SyntaxError,整段程序在跑第一行前 + 就被丢弃,工具调用一并作废。失败的程序可能已经执行过调用,而且同样留下名字,所以重试前要先核对副作用、 + 再改名。新增一小节给出三条规则,错误表补上对应的两行;0.5.6 的沙箱档位小节、示例、步骤与四条轨迹均 + 不变,无 loader 的展示面也保留该规则。一次 127 个程序的会话里,17 个失败程序中有 11 个正是这个错误, + 其中两个是已失败程序的再次发送;本节给教学增加 803 B(8,169 → 8,972 B), + `bench/probes/probe-scope-guidance.mjs` 报告随包发布的修订 9 文本为 5 个示例、其中 4 个不能二次复制 + (`load`、`dirs`、`hits`、`rows`)。修订号 9;行为不变。 + ## [0.5.6] — 2026-11-21 — Patch release - ops-ptc's batching teacher trajectory now teaches sandbox authority: the lesson states the diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index 7aec604c..1a2d65a7 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -102,9 +102,10 @@ external English Markdown file: - **Once per agent entry**: one plain user message rides the first accepted pre-step batch (source kind `user`, no plugin envelope) and enters logged history, so the model sees the whole procedure on the first request: the first-line `// plan:` comment, the dependency split, the load-before-call step (only while dynamic loading publishes the loader) and four copyable trajectories. - The first-round text teaches each rule once (revision 8): the anti-pattern rewrite and the prose that + The first-round text teaches each rule once (revision 9): the anti-pattern rewrite and the prose that repeated the dependency split are gone, the core rules state load-only-missing and recheck-after-a-mode-change - explicitly, and the four trajectories ship byte-identical. + explicitly, the four trajectories ship byte-identical, and one short section states the shared session + scope — a top-level name outlives its program, so a second declaration discards the whole program. - **On a thin program**: when a **root** `run_code` program breaks one of three latched categories, a short notice is queued by that call and appended to the next pre-step as a plain user message. The tool result keeps its value, its content and its existing contexts; the program is never blocked or rewritten. diff --git "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" index ed65eb14..ac5e8f48 100644 --- "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" +++ "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" @@ -72,8 +72,9 @@ opt-in,其解析契约与保守优先级记录在同一文档的 D2 附录, - **每个 agent 一次**:在第一个被接受的 pre-step batch 上追加一条普通用户消息 (source kind `user`、无插件信封),进入会话历史,模型在首个请求即可看到完整范式:首行 `// plan:` 指令、按依赖拆分、先加载再调用(仅动态加载发布加载器时),以及四条可照抄的轨迹。首轮 - 文案按「每条规则只讲一次」精简(修订号 8):反例改写与重复依赖拆分的散文已删,「只加载缺失 API」 - 与「模式变化后重查 SDK」升为核心规则,四条轨迹逐字节保留。 + 文案按「每条规则只讲一次」精简(修订号 9):反例改写与重复依赖拆分的散文已删,「只加载缺失 API」 + 与「模式变化后重查 SDK」升为核心规则,四条轨迹逐字节保留;另有一小节说明会话共享作用域——顶层名字 + 比程序活得久,重声明会让整段程序丢弃。 - **薄程序触发**:**根** `run_code` 程序命中三类锁中的一类时,由该次调用排队、在下一个 pre-step 以普通用户消息 追加一条短提醒。工具结果的值、正文与既有上下文都不变,程序既不阻断也不改写。 - 三类分别是 `bypass`(用 shell 或 `node:fs` 做文件工作)、`plan`(第一物理行没有 `// plan:`)与 `shape` diff --git a/prompts/batching-guidance.md b/prompts/batching-guidance.md index a1880223..5f2c79cd 100644 --- a/prompts/batching-guidance.md +++ b/prompts/batching-guidance.md @@ -34,6 +34,17 @@ the whole program before any line runs. - **danger-full-access**: the widest tier. Never send `sandbox_permissions` or `justification`: no wider target exists, and any value invents one. +## One session, one scope + +While the runtime lives, the session's live scope keeps each program's top-level +`const`, `let`, `class` and `function`, so a second declaration of that name is a +compile-time SyntaxError that discards the whole program before its first line runs, +tool calls included. + +- One name, one program: give each program its own top-level names. +- A failed program may already have run calls: check their effects, then rename before retrying. +- A top-level name is not storage: re-read a value you still need. + ## Step 0 - the plan line Declare the intended calls on the first line of every program: @@ -73,7 +84,8 @@ return { loaded: load.data?.tools, headLines: head.data?.items?.length } Calling an unloaded name throws "tools.ops_read_files is not a function". That is not a reason to fall back to the shell; it is one step spent loading. Loaded names remain -selected in this live scope, and programs need not share a process or local variables. +selected in this live scope; a later program must not assume an earlier program's +values survived, and must not re-declare a name the scope already holds. ## Trajectory 1 - reconnaissance: independent probes in one program @@ -175,4 +187,6 @@ Advance by nextOffset, never by page count, and stop when the result reports com | One failure aborting the batch | Check Outcome status; catch dispatch exceptions and keep e.message | | Shell standing in for a file tool | Use the OPS file API for list, search, read and edit | | ops_read_files before it is loaded | ops_load_sdk in the previous program | +| The same top-level name in a second program | Give the program its own names; a used name is spent | +| Re-sending a failed program with new arguments | Check what it already ran, then rename and retry | | Using e.toolName as the error reason | Use e.message | diff --git a/src/ptc/batching-guidance.ts b/src/ptc/batching-guidance.ts index 4e5ea210..e387d232 100644 --- a/src/ptc/batching-guidance.ts +++ b/src/ptc/batching-guidance.ts @@ -15,7 +15,7 @@ import { literalShellFileWork } from './shell-file-work.js' /** Identity of the guidance in the host message source. */ export const BATCHING_GUIDANCE_ID = 'batching-guidance' /** Revision of the guidance text; bump it when the wording or the file format changes. */ -export const BATCHING_GUIDANCE_REVISION = '8' +export const BATCHING_GUIDANCE_REVISION = '9' /** The shipped guidance file, resolved from this module's own location (src or lib). */ const DEFAULT_GUIDANCE_FILE = new URL('../../prompts/batching-guidance.md', import.meta.url) diff --git a/test/batching-guidance.test.mjs b/test/batching-guidance.test.mjs index 4cce9309..47bc5312 100644 --- a/test/batching-guidance.test.mjs +++ b/test/batching-guidance.test.mjs @@ -20,7 +20,7 @@ await test('the shipped guidance is English Markdown with its host-source identi } assert.equal(BATCHING_GUIDANCE_ID, 'batching-guidance') // A wording or format change is a product change and must bump the revision. - assert.equal(BATCHING_GUIDANCE_REVISION, '8') + assert.equal(BATCHING_GUIDANCE_REVISION, '9') }) await test('the guidance loader returns exactly the shipped Markdown file', () => { @@ -127,10 +127,26 @@ await test('the trimmed lesson keeps the four trajectories and restores the two assert.ok(text.includes('**Recheck the SDK after a mode change**'), 'the post-mode-change recheck is a core rule') }) +await test('the session scope rule reaches every surface that ships the lesson', () => { + // The runtime keeps top-level declarations, so a second declaration discards the + // program before its first line; the rule is not a loader lesson and must survive + // the loader-less rendering that replaces the load section. + const text = loadBatchingGuidance() + assert.ok(text.includes('## One session, one scope'), 'the scope section keeps its heading') + assert.ok(text.includes('compile-time SyntaxError'), 'a redeclaration is named as fatal') + assert.ok(text.includes('The same top-level name in a second program'), 'the table names the trap') + assert.ok(text.includes('Re-sending a failed program with new arguments'), 'the table names the retry trap') + // The 0.5.6 sandbox lesson owns the same slot; a later trim must not drop either. + assert.ok(text.includes('## Sandbox permissions'), 'the sandbox lesson keeps its heading') + const rendered = renderBatchingGuidance(text, { loader: false, resident: ['ops_glob'], loadable: [] }) + assert.ok(rendered.includes('## One session, one scope'), 'the rule is not dropped with the load section') + assert.ok(rendered.includes('## Sandbox permissions'), 'the sandbox tier survives beside the scope rule') +}) + await test('the shipped lesson measures its declared trimmed size', () => { // The trim record in the PR declares this size; a wording edit must bump the // revision and this number together. The 84 B delivery prefix is counted separately. - assert.equal(Buffer.byteLength(loadBatchingGuidance(), 'utf8'), 8169) + assert.equal(Buffer.byteLength(loadBatchingGuidance(), 'utf8'), 8972) }) await test('a scope with the loader receives the shipped file unchanged', () => {