From 61f28e8c90daaf9d7f5c45bd1210e1c2a160fdb9 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:16:39 +0800 Subject: [PATCH 1/9] test(ptc): assert the five legacy config first-round OPS name sets Lock defaults, global dynamicTools:false, row dynamicTools:false, that row plus alwaysLoaded:[stat] and residentTools:false as exact first-round sets, with the old-off-versus-full counterexample and input-sensitivity checks (#985). --- test/ptc-config-truth.test.mjs | 217 +++++++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 test/ptc-config-truth.test.mjs diff --git a/test/ptc-config-truth.test.mjs b/test/ptc-config-truth.test.mjs new file mode 100644 index 00000000..d74d4166 --- /dev/null +++ b/test/ptc-config-truth.test.mjs @@ -0,0 +1,217 @@ +/** The D1 truth table: five legacy config inputs and their first-round OPS names. + * + * This suite locks current publication behaviour, letter by letter, for the five + * inputs the OPS-PTC two-tier contract (#979/#985) must stay compatible with: + * defaults, global dynamicTools:false, row dynamicTools:false, that row plus + * alwaysLoaded:[stat], and residentTools:false. Every row asserts the exact + * first-round ops_* set, so reinterpreting one input as another tier's + * semantics cannot pass silently. The table itself lives in + * docs/design-decisions-20261119.md (中文:docs/设计决策-20261119.md). + */ +import { createScope, scopeOf, scopeParentOf } from '@deepseek-ai/dsh-scope' +import Subprocess from '@deepseek-ai/dsh-subprocess-local' +import Sandbox from '@deepseek-ai/dsh-sandbox-local' +import FileSystem from '@deepseek-ai/dsh-fs-local' +import NodePtc from '@deepseek-ai/dsh-ptc-runtime-node' +import { apply } from '../lib/index.js' +import { resolveMergedConfig, effectiveMode, dynamicForMode } from '../lib/settings/merged.js' +import { resolveModeConfig } from '../lib/modes/config.js' +import { PTC_LOADABLE_TOOLS, PTC_RESIDENT_TOOLS } from '../lib/ptc/config.js' +import { publicToolName } from '../lib/policy.js' +import { bootHost, shutdown } from './lib/host.mjs' +import { prepareStep, executeCall, selectExecutor } from './lib/executor.mjs' +import { assert, test, report, wait, waitFor, withTempDir } from './lib/harness.mjs' +import { WORKER, requireWorker } from './lib/boundary.mjs' + +/** The truth table's right-hand columns, as this suite locks them. */ +const ROW_DEFAULT = ['ops_inspect_local_file', 'ops_load_sdk'] +const ROW_GLOBAL_OFF = ['ops_inspect_local_file'] +const ROW_ROW_OFF = ['ops_inspect_local_file'] +const ROW_ROW_OFF_ALWAYS = ['ops_inspect_local_file', 'ops_stat'] +const ROW_RESIDENT_OFF = ['ops_load_sdk'] + +await test('resolver: the five legacy inputs land where the truth table claims', () => { + const base = resolveMergedConfig() + assert.equal(base['ops-ptc'].enabled, true) + assert.equal(base['ops-ptc'].config.dynamicTools, true) + assert.equal(base['ops-ptc'].config.residentTools, true) + assert.deepEqual(base['ops-ptc'].config.allowedTools, PTC_LOADABLE_TOOLS) + assert.deepEqual(base['ops-ptc'].config.alwaysLoaded, []) + // Row 2: the global switch gates every mode and preserves saved intent. + const global = resolveMergedConfig({ global: { dynamicTools: false } }) + assert.equal(global.global.dynamicTools, false) + for (const mode of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) { + assert.equal(dynamicForMode(global[mode], global.global), false, mode) + assert.equal(effectiveMode(global[mode], global.global).dynamicTools, false, mode) + } + assert.equal('dynamicTools' in (global['ops-ptc'].raw ?? {}), false, 'an absent row key still means on') + // Row 3: a row-level off gates that row only. + const row = resolveMergedConfig({ 'ops-ptc': { dynamicTools: false } }) + assert.equal(effectiveMode(row['ops-ptc'], row.global).dynamicTools, false) + assert.equal(dynamicForMode(row['ops-ptc'], row.global), false) + assert.equal(dynamicForMode(row.standard, row.global), true, 'row-level off leaves other rows on') + // Row 4: alwaysLoaded rides the resident set while the loader is off. + const always = resolveMergedConfig({ 'ops-ptc': { dynamicTools: false, alwaysLoaded: ['stat'] } }) + assert.deepEqual(always['ops-ptc'].config.alwaysLoaded, ['stat']) + assert.deepEqual(always['ops-ptc'].config.allowedTools, PTC_LOADABLE_TOOLS, 'the cap is untouched') + assert.equal(always['ops-ptc'].config.dynamicTools, false) + // Row 5: residentTools is row-local; there is no global resident switch. + const resident = resolveMergedConfig({ 'ops-ptc': { residentTools: false } }) + assert.equal(resident['ops-ptc'].config.residentTools, false) + assert.equal(resident.global.dynamicTools, true) +}) + +await test('the default loadable pool is exactly the sixteen non-resident raw APIs', () => { + assert.deepEqual(PTC_LOADABLE_TOOLS, ['grep', 'glob', 'replace', 'write_file', 'edit_file', 'diff', 'stat', 'copy', + 'move', 'remove', 'read_files', 'read_table', 'read_docx', 'read_pdf', 'batch', 'result']) + assert.deepEqual(PTC_RESIDENT_TOOLS, ['inspect_local_file']) +}) + +await test('today alwaysLoaded escapes the allowedTools cap with no diagnostic (decision item)', () => { + // Current behaviour, locked so a future diagnostic cannot land silently: the + // startup list is validated against the loadable pool only, never against + // allowedTools, and a residentTools:false row keeps the list in config while + // publication drops it. Whether to diagnose is a maintainer decision (#985). + const escaped = resolveModeConfig({ allowedTools: ['grep'], alwaysLoaded: ['stat'] }, 'ops-ptc') + assert.deepEqual(escaped.allowedTools, ['grep']) + assert.deepEqual(escaped.alwaysLoaded, ['stat'], 'stat stays resident though the cap omits it') + const suppressed = resolveModeConfig({ residentTools: false, alwaysLoaded: ['stat'] }, 'ops-ptc') + assert.equal(suppressed.residentTools, false) + assert.deepEqual(suppressed.alwaysLoaded, ['stat'], 'kept in config; publication drops the resident set') +}) + +requireWorker('ptc-config-truth') +let sequence = 0 +const names = (ctx, agent) => ctx.get('tools').definitions(agent, agent.prepared?.toolSet).map(tool => tool.name).sort() +const opsNames = (ctx, agent) => names(ctx, agent).filter(name => name.startsWith('ops_')) +async function step(ctx, agent) { + ctx.emit('session/event', agent.session, { type: 'step/end', turn: 1, step: ++sequence }) + agent.prepared?.dispose(); agent.prepared = await prepareStep(ctx, agent) +} +async function scoped(parent, key, parentKey) { + let result + await parent.plugin({ inject: ['tools'], apply(inner) { + result = createScope(inner, key, parentKey ? { parent: parentKey } : undefined).ctx + } }) + return result +} +async function setup(cwd, mode, switches, global = {}) { + const { ctx } = await bootHost({ withWorkingDirectory: true }) + await ctx.plugin(Subprocess); await ctx.plugin(Sandbox); await ctx.plugin(FileSystem); await ctx.plugin(NodePtc) + ctx.provide('sandboxPolicy', { resolve: ({ session } = {}) => ({ mode: session?.mode ?? 'danger-full-access', workspaceRoot: cwd, sessionId: session?.id }) }) + const presets = new Map(), agents = [] + ctx.provide('agents', { list: () => [...agents] }) + ctx.provide('agentPresets', { composedPreset: inner => presets.get(scopeParentOf(scopeOf(inner))) }) + const config = { global, discovery: { binaryPath: WORKER, required: true } } + for (const id of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) config[id] = { enabled: id === mode, + binaryPath: WORKER, required: true, ...(id === mode ? switches : {}) } + await ctx.plugin({ name: 'ptc-config-truth-fixture', apply: inner => apply(inner, config) }) + const id = 'truth-' + ++sequence, preset = await scoped(ctx, { preset: id }) + presets.set(scopeOf(preset), mode) + for (const name of ['read', 'glob', 'grep']) preset.tools.register({ name, description: name, + parameters: { type: 'object', properties: {} }, output: { schema: { type: 'object' }, render: () => [] }, execute: async () => ({}) }) + const agent = { id, session: { id, mode: 'danger-full-access', header: { cwd }, append() {}, read() { return [] } } } + agent.ctx = await scoped(preset, agent, scopeOf(preset)) + if (mode === 'ptc' || mode === 'ops-ptc') await agent.ctx.plugin({ inject: ['tools', 'systemPrompt'], apply: inner => { selectExecutor(inner, 'ptc') } }) + agents.push(agent) + await ctx.serial('agent/created', { agent }) + await step(ctx, agent) + return { ctx, agent } +} + +/** Boot one input, wait for its first-round anchor, return the settled OPS set. */ +async function firstRoundOps(mode, switches, global, anchor) { + return withTempDir('truth-' + ++sequence, async cwd => { + const { ctx, agent } = await setup(cwd, mode, switches, global) + try { + await waitFor(() => opsNames(ctx, agent).includes(anchor), { what: anchor + ' published' }) + await wait(100) + return opsNames(ctx, agent) + } finally { await shutdown(ctx) } + }) +} + +const rowSets = {} + +await test('row 1 (defaults): first round is inspect + loader; grep loads for the next step', () => withTempDir('truth-row1', async cwd => { + const { ctx, agent } = await setup(cwd, 'ops-ptc', {}) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_inspect_local_file'), { what: 'ops_inspect_local_file published' }) + await wait(100) + rowSets.r1 = opsNames(ctx, agent) + assert.deepEqual(rowSets.r1, ROW_DEFAULT) + const run = await executeCall(ctx, { agent, toolSet: agent.prepared.toolSet, name: 'run_code', + arguments: { description: 'Load grep', code: "const picked = await tools.ops_load_sdk({ tools: ['grep'] })\nreturn { picked }" }, + callId: 'truth-r1-' + ++sequence, signal: new AbortController().signal }) + assert.equal(run.isError, false, JSON.stringify(run)) + assert.equal(run.value.result.picked.status, 'completed', JSON.stringify(run)) + assert.equal(opsNames(ctx, agent).includes('ops_grep'), false, 'loadable-not-loaded: activation is next-step only') + await step(ctx, agent) + assert.ok(opsNames(ctx, agent).includes('ops_grep'), 'grep is callable after the next step') + } finally { await shutdown(ctx) } +})) + +await test('row 2 (global dynamicTools:false): first round is inspect only, no loader', async () => { + rowSets.r2 = await firstRoundOps('ops-ptc', {}, { dynamicTools: false }, 'ops_inspect_local_file') + assert.deepEqual(rowSets.r2, ROW_GLOBAL_OFF) + assert.equal(rowSets.r2.includes('ops_load_sdk'), false) +}) + +await test('row 3 (row dynamicTools:false): the same PTC set as the global switch, row-scoped', async () => { + rowSets.r3 = await firstRoundOps('ops-ptc', { dynamicTools: false }, {}, 'ops_inspect_local_file') + assert.deepEqual(rowSets.r3, ROW_ROW_OFF) + assert.equal(rowSets.r3.includes('ops_load_sdk'), false) + assert.deepEqual(rowSets.r3, rowSets.r2, 'global and row off coincide at the PTC surface; only their scope differs') +}) + +await test('row 4 (row off + alwaysLoaded:[stat]): first round is inspect + stat, no loader', async () => { + rowSets.r4 = await firstRoundOps('ops-ptc', { dynamicTools: false, alwaysLoaded: ['stat'] }, {}, 'ops_inspect_local_file') + assert.deepEqual(rowSets.r4, ROW_ROW_OFF_ALWAYS) + assert.equal(rowSets.r4.includes('ops_load_sdk'), false) +}) + +await test('row 5 (residentTools:false): first round is the loader only; inspect is unloadable', () => withTempDir('truth-row5', async cwd => { + const { ctx, agent } = await setup(cwd, 'ops-ptc', { residentTools: false }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_load_sdk'), { what: 'ops_load_sdk published' }) + await wait(100) + rowSets.r5 = opsNames(ctx, agent) + assert.deepEqual(rowSets.r5, ROW_RESIDENT_OFF) + assert.equal(rowSets.r5.includes('ops_inspect_local_file'), false) + const run = await executeCall(ctx, { agent, toolSet: agent.prepared.toolSet, name: 'run_code', + arguments: { description: 'Probe inspect', code: "const blocked = await tools.ops_load_sdk({ tools: ['inspect_local_file'] })\nreturn { blocked }" }, + callId: 'truth-r5-' + ++sequence, signal: new AbortController().signal }) + assert.equal(run.isError, false, JSON.stringify(run)) + assert.equal(run.value.result.blocked.error.code, 'INVALID_ARGUMENT', 'the resident API is not loadable while residentTools is off') + } finally { await shutdown(ctx) } +})) + +await test('the five recorded sets differ exactly where their one-line inputs differ', () => { + assert.notDeepEqual(rowSets.r1, rowSets.r2, 'the global switch removes the loader') + assert.notDeepEqual(rowSets.r1, rowSets.r3, 'the row switch removes the loader') + assert.notDeepEqual(rowSets.r3, rowSets.r4, 'alwaysLoaded adds stat to the resident set') + assert.notDeepEqual(rowSets.r1, rowSets.r5, 'residentTools:false removes the resident surface') + assert.notDeepEqual(rowSets.r4, rowSets.r5) +}) + +await test('legacy off with allowedTools=[grep,stat] is not a full-tier set', async () => { + // The counterexample #985 requires: old-off and a full selection are different + // inputs. Reading dynamicTools:false as "publish everything allowed" would + // put ops_grep and ops_stat in this first round; today it publishes neither. + const offSet = await firstRoundOps('ops-ptc', { dynamicTools: false, allowedTools: ['grep', 'stat'] }, {}, 'ops_inspect_local_file') + assert.deepEqual(offSet, ROW_ROW_OFF) + assert.equal(offSet.includes('ops_grep'), false) + assert.equal(offSet.includes('ops_stat'), false) + assert.equal(offSet.includes('ops_load_sdk'), false) + const demandSet = await firstRoundOps('ops-ptc', { allowedTools: ['grep', 'stat'] }, {}, 'ops_inspect_local_file') + assert.deepEqual(demandSet, ROW_DEFAULT, 'the same cap on-demand still starts from inspect + loader') + assert.notDeepEqual(offSet, demandSet, 'one flipped line, different first-round set') + assert.equal(demandSet.includes('ops_grep'), false, 'on-demand also keeps grep off the first round') +}) + +await test('the ptc row publishes the same first-round set as ops-ptc', async () => { + const set = await firstRoundOps('ptc', {}, {}, 'ops_inspect_local_file') + assert.deepEqual(set, ROW_DEFAULT) +}) + +report('ptc-config-truth') From 4f1952fd52a5d54b1ac724f56629f996e1730c29 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:16:40 +0800 Subject: [PATCH 2/9] docs(presets): record the legacy OPS-PTC truth table and sign-off items Add the bilingual 2026-11-19 design decision pair with the five-input truth table, the target full-selection row marked unimplemented, and the result exception, global switch and alwaysLoaded-cap conflicts recorded as maintainer sign-off items; cross-link from the configuration reference and record the Unreleased entry (#985). --- docs/CHANGELOG.md | 3 + docs/configuration-reference.md | 6 + docs/design-decisions-20261119.md | 122 ++++++++++++++++++ ...6\241\345\206\263\347\255\226-20261119.md" | 111 ++++++++++++++++ ...15\347\275\256\345\217\202\350\200\203.md" | 5 + 5 files changed, 247 insertions(+) create mode 100644 docs/design-decisions-20261119.md create mode 100644 "docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d66194e1..53c3915e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,9 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Lock the five legacy OPS-PTC configuration inputs as a reviewable truth table: defaults, `global.dynamicTools:false`, row `dynamicTools:false`, that row plus `alwaysLoaded:[stat]`, and `residentTools:false` publish the exact first-round `ops_*` sets now asserted letter by letter by `test/ptc-config-truth.test.mjs` (inspect + loader; inspect; inspect; inspect + stat; loader only — the `ptc` row matches `ops-ptc`). Reading old `dynamicTools:false` as a full selection is pinned as a different input, and the implicit `ops_result` exception, the global-switch gate and `alwaysLoaded`-versus-cap conflicts are recorded as maintainer sign-off items in docs/design-decisions-20261119.md, not implemented. No publication default or behaviour changes (#985). +- 将五种旧 OPS-PTC 配置输入固化为可审阅的真值表:缺省、`global.dynamicTools:false`、行级 `dynamicTools:false`、行关闭加 `alwaysLoaded:[stat]` 与 `residentTools:false` 的首轮 `ops_*` 精确集合由 `test/ptc-config-truth.test.mjs` 逐字断言(inspect+loader;inspect;inspect;inspect+stat;仅 loader——`ptc` 行与 `ops-ptc` 相同)。「把旧 `dynamicTools:false` 解读为全量」被钉为另一档输入;`ops_result` 隐式例外、总开关与门、`alwaysLoaded` 与上限的冲突在 docs/设计决策-20261119.md 记录为维护者签字决定项,未做实现。发布默认值与行为均不变(#985)。 + - Native tool construction and the publication refusal queue no longer attach listeners or lifetime effects to the composition root. The evidence observer and result admission subscribe on the caller's context with global dispatch, and the refusal queue falls back to the caller lifetime when the root refuses an effect. dsh-tui 0.20 refuses `root.events.on` and `root.effect` from a plugin activation, and the native-tools row is one, so every `dsh-tui --resume` aborted with "cannot resume session" before the agent mounted. - 原生工具构造与发布拒绝队列不再向组合根挂监听或生命周期 effect:证据观察者与 result admission 改为在调用方上下文订阅并声明 global 分发,拒绝队列在 root 拒绝 effect 时回落到调用方生命周期。dsh-tui 0.20 会拒绝插件激活期内的 `root.events.on` 与 `root.effect`,而原生工具行正是插件激活,导致每次 `dsh-tui --resume` 都在 agent 挂载前报 "cannot resume session"。 - The Bash tool description now states the parse contract of `stdout`/`stderr` where the model reads it: `text` may be a partial fragment, `truncated` marks one that must not be parsed as complete structured output, and `spillPath` is optional and not a completeness guarantee. The typed output schema is unchanged: repeating the contract there would cost every PTC request without saying anything the tool text does not. Command semantics, exit codes, result values, limits, spill files, permissions and rendering are unchanged. diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index 1d62646f..8ccf1c00 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -85,6 +85,12 @@ Native guidance; `allowedTools`/`alwaysLoaded` accept loadable raw API names onl [Progressive PTC SDK loading](ptc-sdk-loading.md)). The three `spill*` keys bound host subprocess spill retention, described under built-in advanced configuration below. +The exact first-round `ops_*` name sets for the five legacy inputs — defaults, +`global.dynamicTools:false`, row `dynamicTools:false`, that row plus `alwaysLoaded:[stat]`, and +`residentTools:false` — are locked letter by letter in the +[D1 truth table](design-decisions-20261119.md). Old `dynamicTools:false` publishes only the +resident/always-loaded surface and must not be read as a full selection. + ### Batch-call guidance (`batchingGuidance`) Off contributes no extra context. The code default is on for `ops-ptc` only, even when a legacy profile replaces the whole row config without that section; explicit `batchingGuidance: false` still disables it. On, the guidance is delivered in two places, from an diff --git a/docs/design-decisions-20261119.md b/docs/design-decisions-20261119.md new file mode 100644 index 00000000..4b5fb291 --- /dev/null +++ b/docs/design-decisions-20261119.md @@ -0,0 +1,122 @@ +# OPS-PTC legacy configuration: the five-input truth table + +[中文](设计决策-20261119.md) + +Decision date: 2026-11-19. Baseline: `main` at 58f6e28. Card: +[#985](https://github.com/T-Auto/dsh-ops/issues/985) (D1), workflow +[#979](https://github.com/T-Auto/dsh-ops/issues/979), overall plan +[#978](https://github.com/T-Auto/dsh-ops/issues/978). + +This document locks **current** publication behaviour; it implements nothing. Every +row is asserted letter by letter by `node test/run.mjs ptc-config-truth` +(`test/ptc-config-truth.test.mjs`). Byte budgets are deliberately out of scope here — +the V slices measure them. + +## How one input reaches the tool set + +| Stage | Where | What it does | +| --- | --- | --- | +| Row/global keys | `src/settings/merged.ts` `resolveMergedConfig` | Validates the row; defaults `dynamicTools` and `residentTools` to true (`global` has no resident key) | +| AND gate | `effectiveMode` (`src/settings/merged.ts`), `dynamicForMode` | Row `dynamicTools` AND `global.dynamicTools`; the global switch "keeps the operator's intent and simply stops applying" | +| Consumer plan | `src/presets/builtin.ts` `applyResolvedMode` | `gate.resident = config.residentTools`, `gate.dynamic = config.dynamicTools` (the gated value), `plan.ptc = { allowedTools, alwaysLoaded }` | +| Publication | `src/publication/ptc.ts` `mountPtc` | `resident` = `PTC_RESIDENT_TOOLS` + `alwaysLoaded`, empty when `gate.resident === false`; `selectable` = `allowedTools` + `result`, empty when `gate.dynamic === false`; a tool is callable when resident **or** previously selected; `ops_load_sdk` is published only while `gate.dynamic !== false` | + +## The truth table + +Mode `ops-ptc` (the `ptc` row publishes the same first-round set; only +`batchingGuidance` differs). Measured on Linux, `danger-full-access`, native worker +present, real PTC presentation, the host's official `read`/`glob`/`grep` left visible; +`ops_bash` is absent because POSIX rows keep the host shell. + +| # | Input (one line of `dsh-ops` row config) | Callable this step | Loadable, not loaded | Loader | run_code first-program view | +| --- | --- | --- | --- | --- | --- | +| 1 | defaults (no keys) | `ops_inspect_local_file`, `ops_load_sdk` | the sixteen-API pool below | yes | `tools.ops_inspect_local_file` and `tools.ops_load_sdk` work; a `ops_load_sdk({tools:['grep']})` receipt completes but `ops_grep` is callable only in the **next** program | +| 2 | `global: {dynamicTools: false}` | `ops_inspect_local_file` | none | no | `tools.ops_inspect_local_file` works; there is no `tools.ops_load_sdk` | +| 3 | `ops-ptc: {dynamicTools: false}` | `ops_inspect_local_file` | none | no | same set as row 2 (one AND gate, two sides); only the scope differs — see note | +| 4 | row 3 plus `ops-ptc: {alwaysLoaded: [stat]}` | `ops_inspect_local_file`, `ops_stat` | none | no | both bindings work in the first program | +| 5 | `ops-ptc: {residentTools: false}` | `ops_load_sdk` | the sixteen-API pool below | yes | the loader is the only binding; `ops_inspect_local_file` is unreachable — `ops_load_sdk({tools:['inspect_local_file']})` answers `INVALID_ARGUMENT` | +| T | **target, not implemented**: an explicit full selection with `allowedTools: [grep, stat]` | `ops_grep`, `ops_stat` (per #979 §2) | — | no (per #979 §2) | every allowed API callable in the first program; read-only sessions still exclude write APIs | + +The default loadable pool (rows 1 and 5) is exactly: +`ops_grep`, `ops_glob`, `ops_replace`, `ops_write_file`, `ops_edit_file`, `ops_diff`, +`ops_stat`, `ops_copy`, `ops_move`, `ops_remove`, `ops_read_files`, `ops_read_table`, +`ops_read_docx`, `ops_read_pdf`, `ops_batch`, `ops_result` — the sixteen non-resident +raw APIs (`PTC_LOADABLE_TOOLS`). Row T's pool would be `ops_grep`, `ops_stat` **plus +`ops_result`** (see sign-off item 1). + +**Row notes.** + +- Rows 2 and 3 publish the same OPS set because both feed one AND gate. Their inputs are + still different: the global switch also gates the Native discovery entry of + `standard`/`cordis`/`minimal` (`dynamicForMode`), the row switch only its own mode. +- Row 4 shows `dynamicTools:false` does **not** suppress `alwaysLoaded`: the startup list + rides the resident set, with no loader to select the rest of the cap. +- Row 5 shows `residentTools:false` drops the whole resident set, including + `ops_inspect_local_file`, which is not loadable either (it is not in + `PTC_LOADABLE_TOOLS`). +- **Old off is a different input from a full selection** (#985 counterexample): with + `allowedTools: [grep, stat]` and `dynamicTools: false` the first round is + `ops_inspect_local_file` alone — neither `ops_grep` nor `ops_stat` nor the loader. + Reading `dynamicTools:false` as "publish everything allowed" would put `ops_grep` and + `ops_stat` in that first round, changing old users' cost and permission expectations; + #907 implicitly flipped exactly that and #910 reverted it (#979 §4). + +**Read-only and writable instances.** Read-only members of the default pool: +`ops_grep`, `ops_stat`, `ops_glob`, `ops_diff`, `ops_read_files`, `ops_read_table`, +`ops_read_docx`, `ops_read_pdf`. Writable members: `ops_replace`, `ops_write_file`, +`ops_edit_file`, `ops_copy`, `ops_move`, `ops_remove`, `ops_batch`. In a read-only +session `permitted` (`src/publication/ptc.ts`) removes every mutating API from the +loadable pool and a load request for one answers `INVALID_ARGUMENT`; +`ops_result` and `ops_load_sdk` are exempt from the mutation check. No suite here +changes the sandbox mode; D3 covers read-only. + +A program you can read letter by letter: + +```js +// plan: row 1 — load grep now, use it in the next program +const picked = await tools.ops_load_sdk({ tools: ['grep'] }) +return { picked } // picked.status === 'completed'; ops_grep is NOT callable in this program +``` + +```js +// plan: row 4 — stat is resident here, callable in the first program +const r = await tools.ops_stat({ path: 'a' }) +return r.status +``` + +## Maintainer sign-off items — recorded, deliberately not implemented + +1. **The implicit `ops_result` exception.** Today `loadable = [...allowedTools, 'result']` + unconditionally, and `permitted` exempts `result` from the read-only mutation check, so + `ops_result` stays loadable even when the cap omits it (covered today by + `test/resident-switch.test.mjs`, "result exception remains loadable"). With + `dynamicTools:false` (rows 2–4) `ops_result` is **not published at all**. Decision + needed for the full tier: must its first round publish `ops_result` so spilled refs stay + consumable (#979 §3)? Current behaviour is locked; nothing here changes it. +2. **The global switch versus an explicit full selection.** Today `global.dynamicTools` + AND-gates every row's loader, including the PTC one. #979 §4 requires that upgrades, + settings saves and the `global.dynamicTools` switch never pick full **for** the user. + Decision needed: does `global.dynamicTools:false` also suppress an explicitly selected + full row, or does the explicit opt-in win over the bundle switch? +3. **Explicit resident versus the cap.** Today `alwaysLoaded` is validated against the + loadable pool only (`src/ptc/config.ts`), never against `allowedTools`: + `{ allowedTools: [grep], alwaysLoaded: [stat] }` publishes `stat` as resident with no + diagnostic, and `residentTools:false` silently drops the whole resident set including + `alwaysLoaded`. Decision needed: which of these conflicts get a readable diagnostic in + D2, and whether `alwaysLoaded` must be inside `allowedTools`. + +## Scope + +No publication default, no diagnostic and no third user mode is introduced here. Per +#985, disagreements on the sign-off items go to the maintainers on the card before D2/D3 +implement anything. Pure legacy configurations keep the tool sets above before and after +this slice — that is what the suite asserts. + +## Verification + +- `npm run build`, then `node test/run.mjs ptc-config-truth` — 11 passed / 0 skipped / 0 + failed on the baseline (first run matched the code-read prediction on every row). +- Red proof that the assertions bind behaviour, not each other: changing only the row-5 + expected constant in the suite makes exactly that row's check fail + ("1 of 11 check(s) failed"); restoring the constant returns the suite to green. +- `node test/run.mjs resident-switch` — unchanged neighbour suite, see the PR. diff --git "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" new file mode 100644 index 00000000..8fa602f2 --- /dev/null +++ "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" @@ -0,0 +1,111 @@ +# OPS-PTC 旧配置行为:五输入真值表 + +[English](design-decisions-20261119.md) + +决策日期:2026-11-19。基准:`main` 于 58f6e28。任务卡: +[#985](https://github.com/T-Auto/dsh-ops/issues/985)(D1),所属工作流 +[#979](https://github.com/T-Auto/dsh-ops/issues/979),总计划 +[#978](https://github.com/T-Auto/dsh-ops/issues/978)。 + +本文档只锁定**当前**发布行为,不实现任何新东西。每一行都由 +`node test/run.mjs ptc-config-truth`(`test/ptc-config-truth.test.mjs`)逐字断言。 +字节计量刻意不在本卡范围内——V 系列切片负责测量。 + +## 一个输入如何到达工具集合 + +| 阶段 | 位置 | 作用 | +| --- | --- | --- | +| 行/全局键 | `src/settings/merged.ts` `resolveMergedConfig` | 校验行配置;`dynamicTools` 与 `residentTools` 默认 true(`global` 没有常驻键) | +| 与门 | `effectiveMode`(`src/settings/merged.ts`)、`dynamicForMode` | 行 `dynamicTools` 与 `global.dynamicTools` 求与;总开关「保留操作者意图,只是不再生效」 | +| 消费者计划 | `src/presets/builtin.ts` `applyResolvedMode` | `gate.resident = config.residentTools`,`gate.dynamic = config.dynamicTools`(过门后的值),`plan.ptc = { allowedTools, alwaysLoaded }` | +| 发布 | `src/publication/ptc.ts` `mountPtc` | `resident` = `PTC_RESIDENT_TOOLS` + `alwaysLoaded`,`gate.resident === false` 时为空;`selectable` = `allowedTools` + `result`,`gate.dynamic === false` 时为空;工具在常驻**或**已被选中时可调;`ops_load_sdk` 仅在 `gate.dynamic !== false` 时发布 | + +## 真值表 + +以 `ops-ptc` 行为准(`ptc` 行首轮集合相同,仅 `batchingGuidance` 缺省不同)。测量环境: +Linux、`danger-full-access`、native worker 就位、真实 PTC 呈现、宿主官方 +`read`/`glob`/`grep` 保持可见;`ops_bash` 缺席,因为 POSIX 行沿用宿主 shell。 + +| # | 输入(`dsh-ops` 行配置的一行) | 本步可调 | 可加载未加载 | loader | run_code 首轮视角 | +| --- | --- | --- | --- | --- | --- | +| 1 | 缺省(无任何键) | `ops_inspect_local_file`、`ops_load_sdk` | 下方十六个 API 的默认池 | 有 | `tools.ops_inspect_local_file` 与 `tools.ops_load_sdk` 可用;`ops_load_sdk({tools:['grep']})` 回执 completed,但 `ops_grep` 要到**下一个**程序才可调 | +| 2 | `global: {dynamicTools: false}` | `ops_inspect_local_file` | 无 | 无 | `tools.ops_inspect_local_file` 可用;没有 `tools.ops_load_sdk` | +| 3 | `ops-ptc: {dynamicTools: false}` | `ops_inspect_local_file` | 无 | 无 | 与第 2 行同一集合(同一个与门的两侧),差别只在作用域——见注 | +| 4 | 第 3 行再加 `ops-ptc: {alwaysLoaded: [stat]}` | `ops_inspect_local_file`、`ops_stat` | 无 | 无 | 两个绑定首轮即可调用 | +| 5 | `ops-ptc: {residentTools: false}` | `ops_load_sdk` | 下方十六个 API 的默认池 | 有 | loader 是唯一绑定;`ops_inspect_local_file` 整体不可达——`ops_load_sdk({tools:['inspect_local_file']})` 回 `INVALID_ARGUMENT` | +| T | **目标行,未实现**:显式选择全量且 `allowedTools: [grep, stat]` | `ops_grep`、`ops_stat`(依 #979 §2) | — | 无(依 #979 §2) | 获准 API 首轮全部可调;只读会话仍排除写 API | + +默认可加载池(第 1、5 行)恰为: +`ops_grep`、`ops_glob`、`ops_replace`、`ops_write_file`、`ops_edit_file`、`ops_diff`、 +`ops_stat`、`ops_copy`、`ops_move`、`ops_remove`、`ops_read_files`、`ops_read_table`、 +`ops_read_docx`、`ops_read_pdf`、`ops_batch`、`ops_result`——十六个非常驻原始 API +(`PTC_LOADABLE_TOOLS`)。目标行 T 的池应为 `ops_grep`、`ops_stat` **外加 +`ops_result`**(见签字决定项 1)。 + +**逐行注释。** + +- 第 2、3 行发布同一个 OPS 集合,因为它们喂进同一个与门。但输入本身不同:总开关 + 同时关掉 `standard`/`cordis`/`minimal` 的 Native 发现入口(`dynamicForMode`), + 行级开关只影响本行。 +- 第 4 行说明 `dynamicTools:false` **不**压制 `alwaysLoaded`:启动名单随常驻集合发布, + 只是没有 loader 去选择上限中的其余工具。 +- 第 5 行说明 `residentTools:false` 丢掉整个常驻集合,包括 `ops_inspect_local_file`, + 而且它也加载不回来(不在 `PTC_LOADABLE_TOOLS` 里)。 +- **旧行关闭与全量选择是两个不同的输入**(#985 反例):`allowedTools: [grep, stat]` 加 + `dynamicTools: false` 时,首轮只有 `ops_inspect_local_file`——没有 `ops_grep`、没有 + `ops_stat`、没有 loader。把 `dynamicTools:false` 解读成「发布全部获准工具」,就会让 + `ops_grep`、`ops_stat` 出现在这一轮,改变旧用户的成本与权限预期;#907 曾隐式做过 + 这个翻转,#910 随即撤回(#979 §4)。 + +**只读与可写实例。** 默认池中的只读成员:`ops_grep`、`ops_stat`、`ops_glob`、 +`ops_diff`、`ops_read_files`、`ops_read_table`、`ops_read_docx`、`ops_read_pdf`。 +可写成员:`ops_replace`、`ops_write_file`、`ops_edit_file`、`ops_copy`、`ops_move`、 +`ops_remove`、`ops_batch`。只读会话下,`permitted`(`src/publication/ptc.ts`)会把所有 +会变更的 API 从可加载池中剔除,加载请求回 `INVALID_ARGUMENT`;`ops_result` 与 +`ops_load_sdk` 豁免于该变更检查。本卡没有测试切换沙箱模式;只读由 D3 覆盖。 + +一段可以逐字照抄的程序: + +```js +// plan: 第 1 行——现在加载 grep,下一个程序再使用 +const picked = await tools.ops_load_sdk({ tools: ['grep'] }) +return { picked } // picked.status === 'completed';ops_grep 在本程序内仍不可调 +``` + +```js +// plan: 第 4 行——stat 在此行常驻,首轮即可调用 +const r = await tools.ops_stat({ path: 'a' }) +return r.status +``` + +## 维护者签字决定项——如实记录,刻意不实现 + +1. **`ops_result` 的隐式例外。** 今天 `loadable = [...allowedTools, 'result']` 无条件成立, + 且 `permitted` 让 `result` 豁免只读变更检查,因此即使上限没写它,`ops_result` 也始终 + 可加载(现状由 `test/resident-switch.test.mjs` 的 "result exception remains loadable" + 覆盖)。而 `dynamicTools:false`(第 2–4 行)下 `ops_result` **根本不发布**。全量档需要 + 决定:首轮是否发布 `ops_result`,让落盘的 ref 仍然可消费(#979 §3)?当前行为已锁定, + 本文不做任何改动。 +2. **总开关对显式全量选择的作用。** 今天 `global.dynamicTools` 与门控住每一行的 loader, + PTC 行也不例外。#979 §4 要求升级、保存设置和 `global.dynamicTools` 开关都不得替用户 + 选择全量。需要决定:`global.dynamicTools:false` 是否也压制显式选择全量的行,还是显式 + opt-in 优先于总开关? +3. **显式常驻与上限的冲突。** 今天 `alwaysLoaded` 只对照可加载池校验 + (`src/ptc/config.ts`),从不对照 `allowedTools`:`{ allowedTools: [grep], + alwaysLoaded: [stat] }` 会把 `stat` 当常驻发布且没有任何诊断;`residentTools:false` + 则静默丢弃整个常驻集合(含 `alwaysLoaded`)。需要决定:D2 给哪些冲突加可读诊断、 + `alwaysLoaded` 是否必须落在 `allowedTools` 之内。 + +## 范围 + +本文不引入任何发布默认值、诊断或第三种用户模式。依 #985,签字决定项有分歧先在卡上 +请维护者签字,D2/D3 才动手。纯旧配置在本切片前后的工具集合保持上表不变——套件断言 +的正是这一点。 + +## 验证 + +- `npm run build` 后 `node test/run.mjs ptc-config-truth`——基准上 11 通过 / 0 跳过 / + 0 失败(首次运行即与代码阅读预测逐行一致)。 +- 断言绑定行为而非互为同义反复的反向红证:只改套件中第 5 行的期望常量,恰好该行 + 检查失败("1 of 11 check(s) failed");恢复常量后回到全绿。 +- `node test/run.mjs resident-switch`——未改动的邻居套件,见 PR。 diff --git "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" index e3e9281f..cc08a167 100644 --- "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" +++ "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" @@ -58,6 +58,11 @@ OPS 沿用单行插件页:bundle 只插入一个 `dsh-ops` 行。内置预设 见 [PTC SDK 按需加载](PTC按需加载.md))。三个 `spill*` 键作用于宿主 subprocess spill 保留, 语义见下文「内置高级配置」。 +五种旧配置输入——缺省、`global.dynamicTools:false`、行级 `dynamicTools:false`、行关闭加 +`alwaysLoaded:[stat]` 与 `residentTools:false`——的首轮 `ops_*` 名字集合已在 +[D1 真值表](设计决策-20261119.md) 中逐字锁定。旧 `dynamicTools:false` 只发布 +resident/alwaysLoaded 面,不得解读为全量选择。 + ### 批量调用提示(`batchingGuidance`) 关闭时不产生任何额外上下文。代码仅对 `ops-ptc` 默认开启,旧 profile 整行覆盖配置且缺少该节时也保留此默认值;显式 `batchingGuidance: false` 仍可关闭。开启后提示分两处投递,文案来自外置英文 Markdown: From 205cb94a6974f68d2925a3537e1502897045eb7d Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:30:13 +0800 Subject: [PATCH 3/9] test(ptc): add the isolated OPS-PTC entry and routing counter-examples bootEntryGroup mounts the shipped single row on the real agent loop inside each suite's throwaway DSH_HOME and records the environment a measurement must name: plugin commit, host version, platform, the worker sha256 with its staged receipt, sandbox mode, the resolved row switches and the actual agentPreset. enterPtc runs one scripted provider request, proves the top-level transport is run_code before anything executes, and reports a group that lands on Standard, on a native executor or without an executable worker as invalid with its own reason and no comparable output. progressive-loop asserts all of this beside its existing loop checks. --- docs/CHANGELOG.md | 3 + docs/isolated-ptc-entry.md | 73 +++++ ...13\350\257\225\345\205\245\345\217\243.md" | 60 ++++ test/lib/agent-loop.mjs | 14 +- test/lib/ptc-entry.mjs | 292 ++++++++++++++++++ test/progressive-loop.test.mjs | 89 +++++- 6 files changed, 525 insertions(+), 6 deletions(-) create mode 100644 docs/isolated-ptc-entry.md create mode 100644 "docs/\351\232\224\347\246\273\346\265\213\350\257\225\345\205\245\345\217\243.md" create mode 100644 test/lib/ptc-entry.mjs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d66194e1..83f6202a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -15,6 +15,9 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - Replace the `global.forceDefaultOpsPtc` switch with one explicit action: the row's detail page carries a **Make OPS (PTC) the default** button that writes `selectedDefault: ops-ptc` once and answers `applied`/`no-update`/`unavailable`/`failed`. Apply, remount and unload no longer write the default, so a preset chosen on the settings page is not overridden on the next apply, and a saved `global.forceDefaultOpsPtc` key still loads with no effect. - 用一次显式操作取代 `global.forceDefaultOpsPtc` 开关:行详情页有一个「设为 OPS (PTC) 默认」按钮,点一次写 `selectedDefault: ops-ptc`,并回报 `applied`/`no-update`/`unavailable`/`failed`。apply、重挂载与卸载都不再改写默认,设置页选定的预设不会在下一次 apply 被覆盖;已保存的 `global.forceDefaultOpsPtc` 键仍可加载但不再生效。 +- New OPS-PTC acceptance test infrastructure: `test/lib/ptc-entry.mjs` boots one preset group on the real agent loop with the shipped `ops-ptc` row, inside each suite's throwaway `DSH_HOME`, and records the environment a measurement must name — plugin commit, host version, platform, the worker's sha256 with its staged receipt, sandbox mode, the row's resolved switches and the actual `agentPreset`. The minimal case is one scripted provider request proving the top-level transport is `run_code`; a group that lands on Standard, on a native executor, or without an executable worker is reported invalid with its own reason and carries no comparable output, so a wrong configuration can no longer pass as OPS-PTC evidence. `test/progressive-loop.test.mjs` asserts all of this (#990; see `docs/isolated-ptc-entry.md`). +- 新增 OPS-PTC 验收测试基础设施:`test/lib/ptc-entry.mjs` 在每套件独占的临时 `DSH_HOME` 内,用正式 `ops-ptc` 行装配真实 agent loop 的一个组,并记录测量必须交代的环境——插件 commit、宿主版本、平台、worker 的 sha256 及 staged 回执、沙箱模式、行内实际开关与实际 `agentPreset`。最小用例是一次脚本化 provider 请求,先证明顶层传输是 `run_code`;落在 Standard、native executor 或无可执行 worker 上的组按各自原因判「无效」且不带任何可比结果,错误配置不再能冒充 OPS-PTC 证据。`test/progressive-loop.test.mjs` 对此逐项断言(#990,见 `docs/隔离测试入口.md`)。 + ## [0.5.5] — 2026-11-13 — Patch release - A loader-less PTC scope no longer registers the configured `allowedTools` pool: with dynamic loading off nothing could ever select those APIs, and a host assembly without a step tool set declared all sixteen in `tools:sdk`. Resident publication, the default inspect + loader set and selection behavior are unchanged (#887). diff --git a/docs/isolated-ptc-entry.md b/docs/isolated-ptc-entry.md new file mode 100644 index 00000000..052eecd9 --- /dev/null +++ b/docs/isolated-ptc-entry.md @@ -0,0 +1,73 @@ +# Isolated OPS-PTC test entry (#990) + +## Why this entry exists + +A headless comparison can only speak about OPS-PTC if the group it measured +actually entered that routing. Before this entry, the loop fixture assembled a +synthetic `fixture` preset, and the byte probes covered the built-in presets — +so a green number could come from a Standard composition and still look like +OPS-PTC evidence. Issue #980 puts it as the V1 gate: if the actual route is +Standard, the worker is missing, or the routing is unknown, the run is invalid +and must say so. + +## What a group is + +`bootEntryGroup(root, …)` in `test/lib/ptc-entry.mjs` boots one preset group: + +- the real AgentLoop and Session with a scripted provider — no network model + request is ever made; +- the shipped single row (`apply` from the package entry) mounted with only + the target mode enabled, exactly as a profile mounts it, so the ops-ptc row's + real defaults are what the group runs; +- a root directory inside the suite's throwaway `DSH_HOME` — `node test/run.mjs` + creates one home per suite, and the entry refuses any root outside it. + +The environment summary is collected at boot and carried on every report: +plugin commit (read from the git directory, no subprocess), plugin version, host +runtime version, platform (for example `linux-x64`), the worker file with its +sha256, the staged runtime receipt beside it and whether the two digests match, +the sandbox mode and workspace root, the resolved row switches including the +batching guidance default, and the actual `agentPreset` with its mode. The +provider is marked `scripted` and `networkModelRequests` is recorded as 0, +because a mocked provider verifies assembly only and a report must never hide +that. + +## The report contract + +`enterPtc(group)` runs the minimal case first: exactly one scripted provider +request that executes nothing, which is the only request a wrong group ever +earns. Checks run in a fixed order — worker, preset, route, execution — and the +first failure becomes the report's reason. + +- **entered** — the request's top-level surface is exactly `['run_code']` and + the SDK text is present; only then does a second turn invent one fresh probe + file and execute a single program through the worker. The evidence block + records the surface, the preset, the mode, the request counts and the + executed result. +- **invalid** — the report names the reason and carries findings for every + check, but no evidence block and no byte or token numbers of any kind: + - Standard is the deliberate counter-example. It is a real composition, not a + stub, and it fails on `preset` with the route finding showing the native + surface; + - a PTC preset on a native executor fails on `route` — the preset is right, + the transport is not, and the surface may declare no tool at all; + - a row that cannot resolve an executable worker fails at boot on `worker`, + before any request is made. + +Byte and cost accounting is deliberately not collected here; that is the V2 +card's scope (#991). The later cards consume this entry's groups and +environment record instead of rebuilding them. + +## Reproduce + +``` +npm run build +npm run build:ptc +node test/run.mjs progressive-loop +``` + +The filter must select `progressive-loop.test.mjs`, where the four entry +checks live beside the three loop checks. A skip means the worker did not +resolve — run `npm run build:ptc` first; a skip is not a pass. Only freshly +invented fixture files enter these groups; no real session, credential or +conversation text is read or reported. diff --git "a/docs/\351\232\224\347\246\273\346\265\213\350\257\225\345\205\245\345\217\243.md" "b/docs/\351\232\224\347\246\273\346\265\213\350\257\225\345\205\245\345\217\243.md" new file mode 100644 index 00000000..b82771ca --- /dev/null +++ "b/docs/\351\232\224\347\246\273\346\265\213\350\257\225\345\205\245\345\217\243.md" @@ -0,0 +1,60 @@ +# 隔离测试入口(#990) + +## 这个入口为什么存在 + +一次 headless 对比只有在被测的组真正进入 OPS-PTC 路由时,才有资格谈论 +OPS-PTC。在此之前,loop fixture 装配的是合成的 `fixture` 预设,字节探针 +覆盖的是内置预设——一个来自 Standard 组合的绿色数字也可能被当成 OPS-PTC +证据。#980 把它定为 V1 门槛:实际走 Standard、worker 缺失或路由未知时, +这次运行判无效,并且必须把原因说出来。 + +## 一个组是什么 + +`test/lib/ptc-entry.mjs` 里的 `bootEntryGroup(root, …)` 装配一个组: + +- 真实的 AgentLoop 与 Session,脚本化 provider——从不发出真实的网络模型请求; +- 按宿主 profile 的方式挂载正式单行(包入口的 `apply`),只启用目标模式, + 组跑的就是 ops-ptc 行的真实默认值; +- 组的根目录必须位于该套件独占的临时 `DSH_HOME` 内——`node test/run.mjs` + 为每套件建一个 home,入口拒绝任何位于其外的根目录。 + +环境摘要在装配时采集,并附在每份报告上:插件 commit(直接读 git 目录, +不起子进程)、插件版本、宿主运行时版本、平台(例如 `linux-x64`)、 +worker 文件与 sha256、旁边的 staged runtime 回执以及两个摘要是否一致、 +沙箱模式与工作根目录、解析后的行内开关(含批量教学默认值)、以及实际 +`agentPreset` 与其模式。provider 标记为 `scripted`、 +`networkModelRequests` 记为 0——模拟 provider 只能验证装配,报告必须 +把这一点摆在明面上,而不是藏起来。 + +## 报告契约 + +`enterPtc(group)` 先跑最小用例:恰好一次不执行任何调用的脚本化 +provider 请求,而错误的组也只配得到这一次请求。检查按固定顺序执行—— +worker、preset、route、execution——第一个失败项就是报告的 reason。 + +- **entered**——请求的顶层恰好是 `['run_code']` 且 SDK 文本存在;只有这时 + 才进入第二回合,新造一个探针文件并通过 worker 执行一段程序。证据块记录 + 顶层面、preset、mode、请求计数与执行结果。 +- **invalid**——报告给出 reason 与每一项检查的 findings,但不带证据块,也不带 + 任何字节数或 token 数: + - Standard 是故意的反例。它是真实组合而非桩,在 `PRESET` 一项失败, + route 项显示 native 顶层面; + - PTC 预设配 native executor 在 `ROUTE` 一项失败——preset 正确而传输不对, + 顶层面甚至可能一个工具都不声明; + - 解析不到可执行 worker 的行在装配时就以 `WORKER` 失败,不会发出任何请求。 + +字节与成本分账刻意不在本入口采集,那是 V2 卡(#991)的范围。后续卡片 +复用本入口的组与环境记录,而不是重新造一套。 + +## 复现 + +``` +npm run build +npm run build:ptc +node test/run.mjs progressive-loop +``` + +过滤器必须命中 `progressive-loop.test.mjs`,四个入口断言与三个既有 +loop 断言同处一个套件。skip 意味着 worker 没有解析出来——先跑 +`npm run build:ptc`;skip 不是通过。进入这些组的只有新造的样例文件, +不读取、不报告任何真实会话、凭据或对话正文。 diff --git a/test/lib/agent-loop.mjs b/test/lib/agent-loop.mjs index cdcd6fa4..7a91f31e 100644 --- a/test/lib/agent-loop.mjs +++ b/test/lib/agent-loop.mjs @@ -15,12 +15,13 @@ import Sandbox from '@deepseek-ai/dsh-sandbox-local' import FileSystem from '@deepseek-ai/dsh-fs-local' import NodePtc from '@deepseek-ai/dsh-ptc-runtime-node' import { integration } from '../../lib/integration/index.js' +import { apply as applyShippedRow } from '../../lib/index.js' import { assert } from './harness.mjs' import { WORKER } from './boundary.mjs' import { legacyComponent } from './legacy-component.mjs' import { workingDirectoryOver } from './host.mjs' let identity = 0 -export async function bootLoop(root, { ptcMode = false, discoveryConfig = {} } = {}) { +export async function bootLoop(root, { ptcMode = false, discoveryConfig = {}, preset = 'fixture', row } = {}) { const ctx = new Context(); ctx.logger.level = 0 // rc.5 asks the host for the directory a run_code call executes in. ctx.provide('workingDirectory', workingDirectoryOver(root)) @@ -45,11 +46,16 @@ export async function bootLoop(root, { ptcMode = false, discoveryConfig = {} } = inner.on('tools/execute', async (exec, next) => { const result = await next(); results.push({ exec, result }); return result }) inner.on('agent/error', ({ error }) => errors.push(String(error?.stack ?? error))) }) - const ptcFiber = await ctx.plugin(integration({ preset: 'fixture', profile: ptcMode ? 'ptc' : 'native', - binaryPath: WORKER, required: true, ...discoveryConfig })) + // Without a row the composition is the custom-preset integration fixture; with + // one it is the shipped single row exactly as a profile mounts it, so a suite + // can exercise a real mode (e.g. ops-ptc) instead of a synthetic preset. + const composition = row === undefined + ? integration({ preset, profile: ptcMode ? 'ptc' : 'native', binaryPath: WORKER, required: true, ...discoveryConfig }) + : { name: 'dsh-ops-shipped-row', apply: inner => applyShippedRow(inner, row) } + const ptcFiber = await ctx.plugin(composition) await ctx.plugin(Loop, { agents: [], maxParallelToolCalls: 3 }) async function agent(parentAgent) { - const { agent } = await ctx.get('agents').create({ sessionId: 'real-loop-' + ++identity, meta: { cwd: root, agentPreset: 'fixture' }, parentAgent, + const { agent } = await ctx.get('agents').create({ sessionId: 'real-loop-' + ++identity, meta: { cwd: root, agentPreset: preset }, parentAgent, agentOptions: { provider: 'fixture', model: 'scripted' } }) // Exercise a real committed mode event immediately before the first assembly. setSandboxMode(agent.session, 'workspace-write') diff --git a/test/lib/ptc-entry.mjs b/test/lib/ptc-entry.mjs new file mode 100644 index 00000000..f08e5730 --- /dev/null +++ b/test/lib/ptc-entry.mjs @@ -0,0 +1,292 @@ +/** + * The isolated OPS-PTC entry (#990). + * + * A headless comparison is only evidence when the group it measured truly + * entered the OPS-PTC routing: one run_code transport, the shipped ops-ptc row, + * and an executable worker. This module is that guarantee, packaged for the + * later measurement cards (V2-V5) to reuse: + * + * - bootEntryGroup boots one preset group on the real AgentLoop with the + * shipped single row mounted, inside this suite's throwaway DSH_HOME. + * Nothing outside the temp root is read; the probe file the entry inspects + * is freshly invented content, never a real session or credential. + * - collectEnvironment records the machine facts a measurement must be able to + * name: plugin commit, host version, platform, worker digest and staged + * receipt, sandbox mode, the row's resolved switches, and the actual preset. + * The provider is scripted and every report says so: a mocked provider can + * verify assembly only, and no real model request ever leaves the fixture. + * - enterPtc runs the minimal case (one scripted provider request) and + * separates every way the group could quietly prove nothing: a worker that + * cannot execute, a preset that is not the target (Standard is the + * deliberate counter-example), a transport that is not run_code, an + * execution that failed. An invalid report carries its reasons and no + * evidence block at all, so it can never be mistaken for a comparable + * result. Byte and cost accounting is deliberately not collected here; + * that is the V2 card's scope. + * + * @module dsh-ops/test/lib/ptc-entry + */ +import { createHash } from 'node:crypto' +import fs from 'node:fs' +import path from 'node:path' +import { presetOf } from '../../lib/consumer/identity.js' +import { modeOf } from '../../lib/presets/detect.js' +import { MODE_IDS } from '../../lib/presets/types.js' +import { resolveMergedConfig } from '../../lib/settings/merged.js' +import { bootLoop } from './agent-loop.mjs' +import { WORKER } from './boundary.mjs' +import { assert, PACKAGE_ROOT, TEST_HOME } from './harness.mjs' + +/** The preset this entry exists to prove: OPS (PTC), the sandboxed-code routing. */ +export const TARGET_PRESET = 'ops-ptc' + +/** Read a small file as trimmed text, or undefined when it cannot be read. */ +function readText(file) { + try { return fs.readFileSync(file, 'utf8').trim() } catch { return undefined } +} + +/** Parse one JSON file, or undefined when it cannot be read or parsed. */ +function readJson(file) { + try { return JSON.parse(fs.readFileSync(file, 'utf8')) } catch { return undefined } +} + +/** sha256 of one file, or undefined when it cannot be read. */ +function sha256OfFile(file) { + try { return createHash('sha256').update(fs.readFileSync(file)).digest('hex') } catch { return undefined } +} + +/** Whether the worker path names a file this process may execute. */ +function workerExecutable(file) { + try { fs.accessSync(file, fs.constants.X_OK); return true } catch { return false } +} + +/** + * The checked-out commit, read straight from the git directory. + * + * No subprocess is spawned and no environment is trusted: a plain checkout + * (a .git directory), a linked worktree (a .git pointer plus commondir) and a + * detached HEAD all resolve, and anything else honestly reports 'unknown' + * instead of a guess. A suite assertion turns the value into a claim. + * @param {string} [root] - the checkout root; defaults to the package root. + * @returns {string} the 40-hex commit, or 'unknown'. + */ +export function pluginCommit(root = PACKAGE_ROOT) { + const shaOf = value => typeof value === 'string' && /^[0-9a-f]{40}$/i.test(value) ? value.toLowerCase() : undefined + let gitdir = path.join(root, '.git'), common = gitdir + let stat + try { stat = fs.statSync(gitdir) } catch { return 'unknown' } + if (stat.isFile()) { + const pointer = readText(gitdir)?.match(/^gitdir:\s*(\S+)$/m)?.[1] + if (pointer === undefined) return 'unknown' + gitdir = path.resolve(root, pointer) + } + const shared = readText(path.join(gitdir, 'commondir')) + if (shared !== undefined) common = path.resolve(gitdir, shared) + const head = readText(path.join(gitdir, 'HEAD')) + const direct = shaOf(head) + if (direct !== undefined) return direct + const ref = head?.match(/^ref:\s*(\S+)$/)?.[1] + if (ref === undefined) return 'unknown' + for (const base of [gitdir, common]) { + const sha = shaOf(readText(path.join(base, ref))) + if (sha !== undefined) return sha + } + const packed = readText(path.join(common, 'packed-refs')) ?? '' + const escaped = ref.replace(/[^A-Za-z0-9/_.-]/g, '\\$&') + const line = packed.match(new RegExp('^([0-9a-f]{40}) ' + escaped + '$', 'm')) + return line === null ? 'unknown' : line[1] +} + +/** The staged runtime receipt for one platform, or null when nothing is staged. */ +function stagedReceipt(platform) { + const receipt = readJson(path.join(PACKAGE_ROOT, 'runtime', platform, 'provenance.json')) + return receipt === undefined ? null : receipt +} + +/** + * The shipped-row configuration one group boots with: only the target mode is + * enabled, every mode's worker is pinned to the same explicit binary. + * @param {string} mode - the single mode row to enable. + * @param {object} [options] - row inputs. + * @param {string} [options.worker] - the worker binary the row pins. + * @param {Record} [options.switches] - overrides for the enabled mode. + * @returns {Record} the row configuration for the shipped apply. + */ +export function entryRow(mode, { worker = WORKER, switches = {} } = {}) { + const row = { global: {}, discovery: { binaryPath: worker, required: true } } + for (const id of MODE_IDS) { + row[id] = { enabled: id === mode, binaryPath: worker, required: true, ...(id === mode ? switches : {}) } + } + return row +} + +/** + * The environment one group runs in: recorded machine facts, never claims. + * @param {object} options - what the group ran as. + * @param {string} options.root - the group's isolated work directory. + * @param {string} options.preset - the agentPreset the group's agents carry. + * @param {string|undefined} options.mode - the mode row the group enabled. + * @param {string} [options.worker] - the worker binary the row pins. + * @param {Record} options.row - the row configuration booted. + * @param {string} [options.sandboxMode] - the sandbox mode the loop commits. + * @returns {Promise} the environment summary. + */ +export async function collectEnvironment({ root, preset, mode, worker = WORKER, row, sandboxMode = 'workspace-write' } = {}) { + const platform = process.platform + '-' + process.arch + const resolved = resolveMergedConfig(row) + const boot = await import('@deepseek-ai/dsh-app-boot') + const digest = sha256OfFile(worker) + const staged = stagedReceipt(platform) + const state = mode === undefined ? undefined : resolved[mode] + return { + plugin: { commit: pluginCommit(), version: readJson(path.join(PACKAGE_ROOT, 'package.json'))?.version }, + host: { version: typeof boot.getDshRuntimeVersion === 'function' ? boot.getDshRuntimeVersion() : 'unknown' }, + platform, + node: process.version, + worker: { + file: worker, + sha256: digest, + executable: workerExecutable(worker), + staged: staged === null ? null : { target: staged.target, sha256: staged.sha256, version: staged.version }, + matchesStaged: staged === null || digest === undefined ? undefined : digest === staged.sha256, + }, + sandbox: { mode: sandboxMode, workspaceRoot: root }, + dshHome: TEST_HOME, + agentPreset: preset, + mode, + switches: state === undefined ? {} : { + enabled: state.enabled, + residentTools: state.config.residentTools, + dynamicTools: state.config.dynamicTools, + batchingGuidance: state.config.batchingGuidance, + batchingTelemetry: state.config.batchingTelemetry, + hideOfficialTools: state.config.hideOfficialTools, + bash: state.config.bash, + global: resolved.global, + }, + // A mocked provider verifies assembly only; no real model request leaves + // the fixture, and every report carries this marker instead of hiding it. + provider: 'scripted', + networkModelRequests: 0, + } +} + +/** + * Boot one isolated preset group on the real loop with the shipped row. + * + * The group's root must live inside this suite's throwaway DSH_HOME. A boot + * that fails because the row refuses a missing worker is reported as the + * 'worker' failure; any other boot failure is rethrown, because it is a suite + * defect, not an honest environment answer. + * @param {string} root - the isolated work directory (under the suite home). + * @param {object} [options] - group inputs. + * @param {string} [options.preset] - the agentPreset; defaults to the target. + * @param {Record} [options.switches] - mode row overrides. + * @param {'ptc'|'native'} [options.executor] - force the executor; defaults by mode. + * @param {string} [options.worker] - the worker binary; defaults to the test build. + * @param {string} [options.sandboxMode] - recorded with the environment. + * @returns {Promise} the booted group, or an invalid-boot report. + */ +export async function bootEntryGroup(root, { preset = TARGET_PRESET, switches = {}, executor, worker = WORKER, sandboxMode = 'workspace-write' } = {}) { + assert.ok(path.resolve(root).startsWith(path.resolve(TEST_HOME)), + 'the entry only boots groups inside the suite throwaway DSH_HOME, got ' + root) + const mode = MODE_IDS.includes(preset) ? preset : undefined + const row = entryRow(mode ?? 'standard', { worker, switches }) + const environment = await collectEnvironment({ root, preset, mode, worker, row, sandboxMode }) + const ptcPresentation = executor === 'native' ? false : executor === 'ptc' ? true : mode === 'ops-ptc' || mode === 'ptc' + try { + const host = await bootLoop(root, { ptcMode: ptcPresentation, preset, row }) + return { booted: true, host, root, worker, preset, mode, environment, close: () => host.close() } + } catch (error) { + const message = String(error?.message ?? error) + if (/Native worker/.test(message)) { + return { booted: false, reason: 'worker', detail: message, environment } + } + throw error + } +} + +/** The probe file enterPtc invents in the group root before it executes one program. */ +const PROBE_FILE = 'entry-probe.txt' + +/** + * Run the minimal case through the real loop and judge the routing honestly. + * + * Phase one is exactly one scripted provider request and executes nothing: it + * observes the wire surface the group would send. Only a run_code surface + * earns a second turn, which invents one fresh probe file and executes a + * single program through the worker. Checks run in a fixed priority order + * (worker, preset, route, execution) and the first failure becomes the + * report's reason; an invalid report never carries an evidence block. + * @param {object} group - a {@link bootEntryGroup} result. + * @param {object} [options] - validation inputs. + * @param {string} [options.target] - the preset the report demands. + * @returns {Promise} the entry report. + */ +export async function enterPtc(group, { target = TARGET_PRESET } = {}) { + if (group.booted === false) { + return { status: 'invalid', target, reason: group.reason, + findings: { worker: { ok: false, detail: group.detail } }, + environment: group.environment } + } + const { host, worker, environment } = group + const agent = await host.agent() + const identity = { + preset: presetOf(host.ctx, agent), + declared: agent.session?.header?.agentPreset, + mode: modeOf(host.ctx, agent), + } + let surface + const probe = await host.run(agent, [request => { + // A composition that declares no tool at all sends no tools array; that + // absence is data for the route check, not a crash. + surface = { + topLevel: (request.tools ?? []).map(tool => tool.name).sort(), + text: JSON.stringify(request.messages), + } + return [] + }]) + const routeOk = surface.topLevel.length === 1 && surface.topLevel[0] === 'run_code' + && /ops_[a-z0-9_]+:/.test(surface.text) + let execution = { ok: false, requests: 0, result: undefined } + if (routeOk) { + // Freshly invented content only: the probe file is this entry's own fixture. + fs.writeFileSync(path.join(group.root, PROBE_FILE), 'ops-ptc entry probe\n') + const program = '// plan: inspect the freshly made probe file once\n' + + 'const outcome = await tools.ops_inspect_local_file({ path: ' + JSON.stringify(PROBE_FILE) + ' })\n' + + 'return outcome' + const executed = await host.run(agent, [ + () => [{ name: 'run_code', args: { description: 'Prove the OPS-PTC entry executes', code: program } }], []]) + const result = executed.results.find(value => value.exec.name === 'run_code')?.result + execution = { ok: result?.isError === false, requests: executed.requests.length, result } + } + const findings = { + worker: { ok: workerExecutable(worker), file: worker }, + preset: { ok: identity.preset === target, target, actual: identity.preset, declared: identity.declared, mode: identity.mode }, + route: { ok: routeOk, expected: ['run_code'], topLevel: surface.topLevel, transport: routeOk ? 'ptc' : 'unknown-or-native' }, + execution: routeOk + ? { ok: execution.ok, requests: execution.requests } + : { ok: null, attempted: false, note: 'route invalid: nothing was executed' }, + } + const order = ['worker', 'preset', 'route'] + if (routeOk) order.push('execution') + const failed = order.filter(reason => findings[reason].ok !== true) + if (failed.length > 0) { + // The refusal is explicit and carries nothing that could be read as a + // comparable result: no evidence block, no measurement of any kind. + return { status: 'invalid', target, reason: failed[0], findings, environment } + } + return { + status: 'entered', + target, + findings, + evidence: { + topLevel: surface.topLevel, + preset: identity.preset, + mode: identity.mode, + requests: { probe: probe.requests.length, execution: execution.requests }, + executed: execution.result, + }, + environment, + } +} diff --git a/test/progressive-loop.test.mjs b/test/progressive-loop.test.mjs index dfb45959..00f5913f 100644 --- a/test/progressive-loop.test.mjs +++ b/test/progressive-loop.test.mjs @@ -1,9 +1,12 @@ import fs from 'node:fs' +import { createHash } from 'node:crypto' import path from 'node:path' -import { assert, test, report, withTempDir } from './lib/harness.mjs' -import { requireWorker } from './lib/boundary.mjs' +import { assert, test, report, withTempDir, PACKAGE_ROOT } from './lib/harness.mjs' +import { requireWorker, WORKER } from './lib/boundary.mjs' import { setSandboxMode } from '@deepseek-ai/dsh-sandbox-policy' import { bootLoop } from './lib/agent-loop.mjs' +import { workerName } from '../lib/execution/runtime.js' +import { bootEntryGroup, enterPtc } from './lib/ptc-entry.mjs' requireWorker('progressive-loop') const names = request => request.tools.map(tool => tool.name) const text = request => JSON.stringify(request.messages) @@ -83,4 +86,86 @@ await test('real committed authority changes cancel pending activation while alw assert.equal(fs.readFileSync(path.join(root, 'note'), 'utf8'), 'unchanged') } finally { await host.close() } })) + +/** Card #990: the isolated OPS-PTC entry. Every group boots on its own throwaway + * root under this suite's DSH_HOME and is judged by test/lib/ptc-entry.mjs. */ +await test('the isolated entry truly routes one OPS-PTC request and records its environment', () => withTempDir('ptc-entry-target', async root => { + fs.writeFileSync(path.join(root, 'entry-note'), 'fixture line\n') + const group = await bootEntryGroup(root) + try { + assert.equal(group.booted, true, 'the target group must boot on the shipped ops-ptc row') + const report = await enterPtc(group) + assert.equal(report.status, 'entered', JSON.stringify(report.findings)) + assert.equal(report.target, 'ops-ptc') + assert.deepEqual(report.evidence.topLevel, ['run_code'], JSON.stringify(report.evidence)) + assert.equal(report.findings.preset.actual, 'ops-ptc') + assert.equal(report.findings.preset.mode, 'ops-ptc') + assert.equal(report.environment.agentPreset, 'ops-ptc') + assert.match(report.environment.plugin.commit, /^[0-9a-f]{40}$/) + assert.match(report.environment.host.version, /^\d+\.\d+\.\d+/) + assert.equal(report.environment.platform, process.platform + '-' + process.arch) + assert.equal(report.environment.sandbox.mode, 'workspace-write') + assert.equal(report.environment.provider, 'scripted', 'the scripted provider is marked on the report, never silent') + assert.equal(report.environment.networkModelRequests, 0) + // The shipped row's real defaults are recorded from the resolved config, not assumed. + assert.equal(report.environment.switches.batchingGuidance, true) + assert.equal(report.environment.switches.residentTools, true) + assert.equal(report.environment.switches.dynamicTools, true) + // The worker fact: the recorded digest is the digest of the binary this group used. + assert.match(report.environment.worker.sha256, /^[0-9a-f]{64}$/) + assert.equal(report.environment.worker.sha256, createHash('sha256').update(fs.readFileSync(WORKER)).digest('hex')) + const staged = report.environment.worker.staged + if (staged !== null) { + assert.equal(staged.target, report.environment.platform) + assert.equal(staged.sha256, createHash('sha256').update(fs.readFileSync( + path.join(PACKAGE_ROOT, 'runtime', report.environment.platform, workerName()))).digest('hex'), + 'the recorded staged receipt matches the staged binary') + } + // The resident SDK really answered through the worker; a fresh fixture file, nothing real. + const executed = report.evidence.executed + assert.equal(executed.isError, false, JSON.stringify(executed)) + assert.equal(executed.value.result.status, 'completed') + } finally { await group.close() } +})) + +await test('Standard routing is the deliberate counter-example: invalid with a reason, never comparable', () => withTempDir('ptc-entry-standard', async root => { + const group = await bootEntryGroup(root, { preset: 'standard' }) + try { + assert.equal(group.booted, true) + const report = await enterPtc(group) + assert.equal(report.status, 'invalid') + assert.equal(report.reason, 'preset', JSON.stringify(report.findings)) + assert.equal(report.findings.preset.actual, 'standard') + assert.equal(report.findings.preset.mode, 'standard') + assert.equal(report.findings.route.ok, false) + assert.equal(report.findings.route.topLevel.includes('run_code'), false) + // The refusal is explicit and carries no surface that could be mistaken for a result. + assert.equal('evidence' in report, false) + assert.deepEqual(Object.keys(report).sort(), ['environment', 'findings', 'reason', 'status', 'target']) + assert.equal(JSON.stringify(report).includes('Bytes'), false) + } finally { await group.close() } +})) + +await test('a PTC preset on a native executor is route-unknown, not a quiet pass', () => withTempDir('ptc-entry-route', async root => { + const group = await bootEntryGroup(root, { executor: 'native' }) + try { + assert.equal(group.booted, true) + const report = await enterPtc(group) + assert.equal(report.status, 'invalid') + assert.equal(report.reason, 'route', JSON.stringify(report.findings)) + assert.equal(report.findings.preset.ok, true, 'the preset is right; the transport is what is unknown') + assert.equal(report.findings.route.topLevel.includes('run_code'), false) + assert.equal('evidence' in report, false) + } finally { await group.close() } +})) + +await test('a group without an executable worker reports invalid before any request', () => withTempDir('ptc-entry-worker', async root => { + const group = await bootEntryGroup(root, { worker: path.join(root, 'no-such-worker') }) + assert.equal(group.booted, false) + const report = await enterPtc(group) + assert.equal(report.status, 'invalid') + assert.equal(report.reason, 'worker') + assert.match(report.findings.worker.detail, /Native worker/) + assert.equal('evidence' in report, false) +})) report('progressive-loop') From 12e3ac5c781d70d16b460310c3eef879355fd255 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 11:50:01 +0800 Subject: [PATCH 4/9] docs(presets): append the D1 truth-table supplementary audit Record four verified facts alongside the delivered truth table, changing no row or existing assertion: the custom integration path rejects alwaysLoaded outside allowedTools (options.ts:100) while built-in rows accept it silently; the settings-page discovery-entry copy versus the typed ops_load_sdk the PTC row actually publishes; the legal custom ptc-profile integration misdiagnosed PUBLICATION_UNAVAILABLE: ops_result (fix owned by #988); and the loader 1-5-per-call cap with the uncapped resident startup list. Add the two matching assertions to test/ptc-config-truth.test.mjs (13 checks). --- docs/CHANGELOG.md | 2 + docs/design-decisions-20261119.md | 49 +++++++++++++++++++ ...6\241\345\206\263\347\255\226-20261119.md" | 40 +++++++++++++++ test/ptc-config-truth.test.mjs | 48 +++++++++++++++++- 4 files changed, 138 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 53c3915e..f1d41fa0 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Record the D1 supplementary audit for the legacy-config truth table: the custom PTC integration path rejects `alwaysLoaded` outside `allowedTools` (`src/consumer/options.ts`) while built-in rows accept it silently; the settings-page "discovery entry" copy is recorded verbatim next to the typed `ops_load_sdk` the PTC row actually publishes; a legal custom `profile: 'ptc'` integration is verified to be misdiagnosed `PUBLICATION_UNAVAILABLE: ops_result` (fix owned by #988); and one `ops_load_sdk` call is locked at 1-5 raw names (four calls minimum for the sixteen-name pool, resident lists uncapped). `test/ptc-config-truth.test.mjs` gains two assertions (13 checks); docs/design-decisions-20261119.md gains the matching append-only sections. No behaviour changes (#985). +- 补录 D1 真值表的追加审计:自定义 PTC integration 路径拒绝落在 `allowedTools` 之外的 `alwaysLoaded`(`src/consumer/options.ts`),内置行却静默接受;设置页「发现入口」文案逐字照录,并注明 PTC 行实际发布的是类型化 `ops_load_sdk`;实测合法自定义 `profile: 'ptc'` integration 会被误诊 `PUBLICATION_UNAVAILABLE: ops_result`(修复归 #988);一次 `ops_load_sdk` 调用锁定为 1-5 个原始名(十六名池至少四次调用,常驻名单无数量上限)。`test/ptc-config-truth.test.mjs` 新增两项断言(13 项检查),docs/设计决策-20261119.md 同步追加对应小节。行为不变(#985)。 - Lock the five legacy OPS-PTC configuration inputs as a reviewable truth table: defaults, `global.dynamicTools:false`, row `dynamicTools:false`, that row plus `alwaysLoaded:[stat]`, and `residentTools:false` publish the exact first-round `ops_*` sets now asserted letter by letter by `test/ptc-config-truth.test.mjs` (inspect + loader; inspect; inspect; inspect + stat; loader only — the `ptc` row matches `ops-ptc`). Reading old `dynamicTools:false` as a full selection is pinned as a different input, and the implicit `ops_result` exception, the global-switch gate and `alwaysLoaded`-versus-cap conflicts are recorded as maintainer sign-off items in docs/design-decisions-20261119.md, not implemented. No publication default or behaviour changes (#985). - 将五种旧 OPS-PTC 配置输入固化为可审阅的真值表:缺省、`global.dynamicTools:false`、行级 `dynamicTools:false`、行关闭加 `alwaysLoaded:[stat]` 与 `residentTools:false` 的首轮 `ops_*` 精确集合由 `test/ptc-config-truth.test.mjs` 逐字断言(inspect+loader;inspect;inspect;inspect+stat;仅 loader——`ptc` 行与 `ops-ptc` 相同)。「把旧 `dynamicTools:false` 解读为全量」被钉为另一档输入;`ops_result` 隐式例外、总开关与门、`alwaysLoaded` 与上限的冲突在 docs/设计决策-20261119.md 记录为维护者签字决定项,未做实现。发布默认值与行为均不变(#985)。 diff --git a/docs/design-decisions-20261119.md b/docs/design-decisions-20261119.md index 4b5fb291..4e5893a3 100644 --- a/docs/design-decisions-20261119.md +++ b/docs/design-decisions-20261119.md @@ -104,6 +104,51 @@ return r.status diagnostic, and `residentTools:false` silently drops the whole resident set including `alwaysLoaded`. Decision needed: which of these conflicts get a readable diagnostic in D2, and whether `alwaysLoaded` must be inside `allowedTools`. + Supplementary contrast fact (append-only audit, same day): the **custom + integration path** is stricter — `src/consumer/options.ts:100` rejects a custom PTC + config whose `alwaysLoaded` escapes `allowedTools` with + `alwaysLoaded must be in allowedTools`, while the built-in rows accept the same + shape silently (locked above by the suite's "escapes the allowedTools cap" check). + The diagnostic inconsistency is owned by the D4 card (#988); D1 records the current + state only. +4. **`ops_result` misdiagnosed as unpublished on a legal custom PTC integration.** + `src/integration/diagnostics.ts` lists `'result'` among the first-round expected + visible names of every PTC integration scope, but `result` is selectable, not + resident (`PTC_RESIDENT_TOOLS` is `inspect_local_file` alone), so it becomes + visible only after a loader call selects it. A perfectly legal custom + `profile: 'ptc'` integration is therefore diagnosed `degraded` with + `PUBLICATION_UNAVAILABLE: ops_result` while its only gap is an API the loader has + not selected yet — verified by booting such an integration; the suite's + "misdiagnosed PUBLICATION_UNAVAILABLE" check locks it. Built-in mode rows never + mount diagnostics at all (`mountDiagnostics` runs only from `mountIntegration`), + so the misreport is reachable only through the custom-integration path. The fix is + owned by the D4 card (#988); D1 records the current state only. + +## Settings-page copy — recorded verbatim + +The settings page and its row switch describe the dynamic-tools switch as a +"discovery entry". The shipped strings, recorded verbatim from +`src/client/locales.ts`: + +| Key | English | 中文 | +| --- | --- | --- | +| `dynamicToolsHint` | "When off, no preset offers the discovery entry." | "关闭后没有任何预设提供发现入口。" | +| `swDynamicToolsHint` | "Offers the discovery entry to this preset. Applies only while the bundle switch is on." | "为该预设提供发现入口。仅在总开关开启时生效。" | + +What the PTC row actually publishes for that entry is the typed loader +`ops_load_sdk` (truth-table rows 1 and 5), not a discovery-style tool: the wording +and the artifact differ. Unifying the wording is the D5 slice of #979; this card +records the gap only. + +## Loader batching — recorded current behaviour + +One `ops_load_sdk` call accepts 1–5 exact raw API names (`requestedApis` in +`src/projections/ptc/loader.ts` — `tools must contain 1-5 exact raw API names`), so +filling the sixteen-name default pool takes at least four calls. `alwaysLoaded` and +the rest of the resident set carry no count cap: `parsePtcConfig` +(`src/ptc/config.ts`) validates each name against the loadable pool, never the +list's length, and resident APIs publish without any loader call. Locked by the +suite's "loader batches 1-5 names" check. ## Scope @@ -120,3 +165,7 @@ this slice — that is what the suite asserts. expected constant in the suite makes exactly that row's check fail ("1 of 11 check(s) failed"); restoring the constant returns the suite to green. - `node test/run.mjs resident-switch` — unchanged neighbour suite, see the PR. +- Supplementary audit (append-only, same day): two checks were added — the loader's + 1–5-per-call cap with the count-cap-free resident startup list, and the legal + custom PTC integration misdiagnosed `PUBLICATION_UNAVAILABLE: ops_result`. The + suite now reports 13 passed / 0 skipped / 0 failed on the baseline. diff --git "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" index 8fa602f2..d28fd717 100644 --- "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" +++ "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" @@ -95,6 +95,43 @@ return r.status alwaysLoaded: [stat] }` 会把 `stat` 当常驻发布且没有任何诊断;`residentTools:false` 则静默丢弃整个常驻集合(含 `alwaysLoaded`)。需要决定:D2 给哪些冲突加可读诊断、 `alwaysLoaded` 是否必须落在 `allowedTools` 之内。 + 补录对比事实(同日追加,只记录不实现):**自定义 integration 路径**更严—— + `src/consumer/options.ts:100` 对 `alwaysLoaded` 不在 `allowedTools` 内的自定义 PTC + 配置直接抛 `alwaysLoaded must be in allowedTools`,而内置行对同一形状静默接受 + (已由套件「escapes the allowedTools cap」检查锁定)。该诊断不一致已由 D4 卡 + (#988)认领修复;D1 只记录现状。 +4. **合法自定义 PTC integration 被误诊 `ops_result` 未发布。** + `src/integration/diagnostics.ts` 把 `'result'` 列入每个 PTC integration 作用域的 + 首轮期望可见名,但 `result` 是可选加载而非常驻(`PTC_RESIDENT_TOOLS` 仅 + `inspect_local_file` 一个),只有 loader 选中后才可见。于是完全合法的自定义 + `profile: 'ptc'` integration 会被诊断为 `degraded`,报 + `PUBLICATION_UNAVAILABLE: ops_result`,而它唯一的「缺口」是一个尚未被加载的 + API——已实测启动这样一个 integration 证实,并由套件的 + 「misdiagnosed PUBLICATION_UNAVAILABLE」检查锁定。内置模式行根本不挂诊断 + (`mountDiagnostics` 只在 `mountIntegration` 中运行),因此该误报只可能出现在 + 自定义 integration 路径上。修复已由 D4 卡(#988)认领;D1 只记录现状。 + +## 设置页文案——如实照录 + +设置页与行开关把 dynamic-tools 开关描述为「发现入口」。已发布文案照录自 +`src/client/locales.ts`: + +| 键 | 英文 | 中文 | +| --- | --- | --- | +| `dynamicToolsHint` | "When off, no preset offers the discovery entry." | "关闭后没有任何预设提供发现入口。" | +| `swDynamicToolsHint` | "Offers the discovery entry to this preset. Applies only while the bundle switch is on." | "为该预设提供发现入口。仅在总开关开启时生效。" | + +PTC 行实际为该入口发布的是类型化 loader `ops_load_sdk`(真值表第 1、5 行), +并非发现式工具:措辞与产物存在差异。统一措辞属 #979 的 D5 切片;本卡只记录差异。 + +## Loader 分批——如实记录现状 + +一次 `ops_load_sdk` 调用接受 1–5 个精确原始 API 名 +(`src/projections/ptc/loader.ts` 的 `requestedApis`—— +`tools must contain 1-5 exact raw API names`),因此装满十六名默认池至少需要四次 +调用。`alwaysLoaded` 与其余常驻集合没有数量上限:`parsePtcConfig` +(`src/ptc/config.ts`)只逐名对照可加载池校验,从不校验名单长度,且常驻 API +无需任何 loader 调用即发布。由套件「loader batches 1-5 names」检查锁定。 ## 范围 @@ -109,3 +146,6 @@ return r.status - 断言绑定行为而非互为同义反复的反向红证:只改套件中第 5 行的期望常量,恰好该行 检查失败("1 of 11 check(s) failed");恢复常量后回到全绿。 - `node test/run.mjs resident-switch`——未改动的邻居套件,见 PR。 +- 补录审计(同日追加):新增两项检查——loader 单次 1–5 上限与常驻启动名单无数量 + 上限、合法自定义 PTC integration 被误诊 `PUBLICATION_UNAVAILABLE: ops_result`。 + 套件在基准上现为 13 通过 / 0 跳过 / 0 失败。 diff --git a/test/ptc-config-truth.test.mjs b/test/ptc-config-truth.test.mjs index d74d4166..f7bd9e83 100644 --- a/test/ptc-config-truth.test.mjs +++ b/test/ptc-config-truth.test.mjs @@ -16,7 +16,9 @@ import NodePtc from '@deepseek-ai/dsh-ptc-runtime-node' import { apply } from '../lib/index.js' import { resolveMergedConfig, effectiveMode, dynamicForMode } from '../lib/settings/merged.js' import { resolveModeConfig } from '../lib/modes/config.js' -import { PTC_LOADABLE_TOOLS, PTC_RESIDENT_TOOLS } from '../lib/ptc/config.js' +import { PTC_LOADABLE_TOOLS, PTC_RESIDENT_TOOLS, parsePtcConfig } from '../lib/ptc/config.js' +import { requestedApis } from '../lib/projections/ptc/loader.js' +import { integration, diagnose } from '../lib/integration/index.js' import { publicToolName } from '../lib/policy.js' import { bootHost, shutdown } from './lib/host.mjs' import { prepareStep, executeCall, selectExecutor } from './lib/executor.mjs' @@ -214,4 +216,48 @@ await test('the ptc row publishes the same first-round set as ops-ptc', async () assert.deepEqual(set, ROW_DEFAULT) }) +await test('the loader batches 1-5 names per call; the resident startup list has no count cap', () => { + // Current behaviour, locked so a silent cap change cannot pass: one load_sdk call + // accepts 1-5 exact raw names, so the sixteen-name pool needs at least four calls, + // while alwaysLoaded publishes any validated list with no length limit. + const empty = requestedApis({ tools: [] }, PTC_LOADABLE_TOOLS) + assert.equal(typeof empty, 'string', 'an empty request is rejected') + assert.ok(String(empty).includes('1-5'), 'the rejection names the 1-5 contract') + assert.equal(typeof requestedApis({ tools: PTC_LOADABLE_TOOLS.slice(0, 6) }, PTC_LOADABLE_TOOLS), 'string', 'a six-name request is rejected') + assert.deepEqual(requestedApis({ tools: ['grep', 'grep', 'stat'] }, PTC_LOADABLE_TOOLS), ['grep', 'stat'], 'duplicates collapse; a legal pair loads') + assert.equal(Math.ceil(PTC_LOADABLE_TOOLS.length / 5), 4, 'sixteen names need at least four loader calls') + const full = parsePtcConfig({ allowedTools: PTC_LOADABLE_TOOLS, alwaysLoaded: PTC_LOADABLE_TOOLS }) + assert.deepEqual(full.alwaysLoaded, PTC_LOADABLE_TOOLS, 'no count cap on the startup list') + assert.deepEqual(full.allowedTools, PTC_LOADABLE_TOOLS) +}) + +await test('a legal custom ptc-profile integration is misdiagnosed PUBLICATION_UNAVAILABLE: ops_result (decision item 4)', () => withTempDir('truth-diag-' + ++sequence, async cwd => { + // Current behaviour, locked so the D4 fix (#988) cannot land silently: diagnostics + // lists ops_result among first-round expected names, but result is selectable, not + // resident, so a legal custom PTC integration reads degraded while its only gap is + // an API the loader has not selected yet. + const { ctx } = await bootHost({ withWorkingDirectory: true }) + try { + await ctx.plugin(Subprocess); await ctx.plugin(Sandbox); await ctx.plugin(FileSystem); await ctx.plugin(NodePtc) + ctx.provide('sandboxPolicy', { resolve: ({ session } = {}) => ({ mode: session?.mode ?? 'danger-full-access', workspaceRoot: cwd, sessionId: session?.id }) }) + ctx.provide('agentPresets', { composedPreset: () => undefined }) + const agents = [] + ctx.provide('agents', { list: () => [...agents] }) + await ctx.plugin(integration({ preset: 'truth-diag-ptc', profile: 'ptc', binaryPath: WORKER, required: true })) + const id = 'truth-diag-' + ++sequence + const agent = { id, session: { id, mode: 'danger-full-access', header: { cwd, agentPreset: 'truth-diag-ptc' } } } + await ctx.plugin({ inject: ['tools'], apply(inner) { agent.ctx = createScope(inner, agent).ctx } }) + await agent.ctx.plugin({ inject: ['tools', 'systemPrompt'], apply: inner => { selectExecutor(inner, 'ptc') } }) + agents.push(agent) + await ctx.serial('agent/created', { agent }) + agent.prepared = await prepareStep(ctx, agent) + await waitFor(() => opsNames(ctx, agent).includes('ops_inspect_local_file'), { what: 'ops_inspect_local_file published' }) + await wait(100) + const d = await diagnose(ctx, agent) + assert.equal(d.state, 'degraded', JSON.stringify(d.issues)) + assert.ok(d.issues.some(row => row.code === 'PUBLICATION_UNAVAILABLE' && row.message.includes('ops_result')), + 'the only missing name is the selectable-but-not-resident result API') + } finally { await shutdown(ctx) } +})) + report('ptc-config-truth') From c3f0fdd00c193728fdb1ffd9ccfb3a48b0f59800 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:15:48 +0800 Subject: [PATCH 5/9] feat(presets): parse an explicit PTC disclosure policy key Add the YAML-only disclosurePolicy key ('on-demand' | 'full', default 'on-demand' = the legacy path) to the ptc and ops-ptc rows: PTC config validation, mode-row routing with a readable rejection on non-PTC rows, and settings serialization that preserves an explicit choice and never writes the key for the user. Resolve the D1 sign-off item on precedence conservatively per the workflow contract: while global.dynamicTools is off the full tier is not applied and the saved choice stays untouched. Publication of the full tier remains the D3 slice; no publication behaviour or settings UI changes. --- docs/CHANGELOG.md | 2 + docs/configuration-reference.md | 5 +- docs/design-decisions-20261119.md | 45 ++++++ ...6\241\345\206\263\347\255\226-20261119.md" | 33 ++++ ...15\347\275\256\345\217\202\350\200\203.md" | 4 +- src/presets/config.ts | 16 +- src/ptc/config.ts | 18 ++- src/settings/merged.ts | 6 + test/ptc-disclosure-policy.test.mjs | 142 ++++++++++++++++++ 9 files changed, 263 insertions(+), 8 deletions(-) create mode 100644 test/ptc-disclosure-policy.test.mjs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f1d41fa0..3a8cde35 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Parse an explicit PTC disclosure policy without changing any legacy behaviour: the `ptc` and `ops-ptc` rows accept a YAML-only `disclosurePolicy` key, `'on-demand'` (default, the legacy progressive path) or `'full'`. Illegal values are errors naming the key and the received value or type, and non-PTC rows reject the key with a readable diagnostic. The choice survives read → save-unrelated-setting → read through the settings serializer, and no upgrade, settings save or `global.dynamicTools` flip ever writes or erases it. The D1 sign-off item on precedence is resolved conservatively: while `global.dynamicTools` is off the full tier is not applied — the row keeps its legacy resident/alwaysLoaded surface — and the saved choice re-applies when the switch returns on; the row-level `dynamicTools` keeps its loader-only meaning because the full tier publishes no loader. Publishing the full tier remains the D3 slice; no publication behaviour or settings UI changes (#986). +- 解析显式 PTC 披露策略且不改变任何旧行为:`ptc` 与 `ops-ptc` 行接受 YAML-only 的 `disclosurePolicy` 键,取值 `'on-demand'`(缺省,即旧渐进路径)或 `'full'`。非法值报错点名键与收到的值或类型,非 PTC 行以可读诊断拒绝该键。选择经设置序列化器在「读取 → 保存无关设置 → 再读取」后不丢,升级、保存设置与 `global.dynamicTools` 翻转都不写入或抹掉它。D1 签字项中优先级问题按保守取向解决:`global.dynamicTools` 关闭期间全量档不生效——行保持旧 resident/alwaysLoaded 面——开关回开后已存选择重新生效;行级 `dynamicTools` 保持只管 loader 的含义,因为全量档不发布 loader。全量档的发布仍归 D3 切片;发布行为与设置 UI 均不变(#986)。 - Record the D1 supplementary audit for the legacy-config truth table: the custom PTC integration path rejects `alwaysLoaded` outside `allowedTools` (`src/consumer/options.ts`) while built-in rows accept it silently; the settings-page "discovery entry" copy is recorded verbatim next to the typed `ops_load_sdk` the PTC row actually publishes; a legal custom `profile: 'ptc'` integration is verified to be misdiagnosed `PUBLICATION_UNAVAILABLE: ops_result` (fix owned by #988); and one `ops_load_sdk` call is locked at 1-5 raw names (four calls minimum for the sixteen-name pool, resident lists uncapped). `test/ptc-config-truth.test.mjs` gains two assertions (13 checks); docs/design-decisions-20261119.md gains the matching append-only sections. No behaviour changes (#985). - 补录 D1 真值表的追加审计:自定义 PTC integration 路径拒绝落在 `allowedTools` 之外的 `alwaysLoaded`(`src/consumer/options.ts`),内置行却静默接受;设置页「发现入口」文案逐字照录,并注明 PTC 行实际发布的是类型化 `ops_load_sdk`;实测合法自定义 `profile: 'ptc'` integration 会被误诊 `PUBLICATION_UNAVAILABLE: ops_result`(修复归 #988);一次 `ops_load_sdk` 调用锁定为 1-5 个原始名(十六名池至少四次调用,常驻名单无数量上限)。`test/ptc-config-truth.test.mjs` 新增两项断言(13 项检查),docs/设计决策-20261119.md 同步追加对应小节。行为不变(#985)。 - Lock the five legacy OPS-PTC configuration inputs as a reviewable truth table: defaults, `global.dynamicTools:false`, row `dynamicTools:false`, that row plus `alwaysLoaded:[stat]`, and `residentTools:false` publish the exact first-round `ops_*` sets now asserted letter by letter by `test/ptc-config-truth.test.mjs` (inspect + loader; inspect; inspect; inspect + stat; loader only — the `ptc` row matches `ops-ptc`). Reading old `dynamicTools:false` as a full selection is pinned as a different input, and the implicit `ops_result` exception, the global-switch gate and `alwaysLoaded`-versus-cap conflicts are recorded as maintainer sign-off items in docs/design-decisions-20261119.md, not implemented. No publication default or behaviour changes (#985). diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index 8ccf1c00..9152af9f 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -74,6 +74,7 @@ the row's status card, which sets it once when pressed. Generation and the manua | `hidePwsh` | true | Hide the official Windows shell while Bash is available; disabled entirely when `global.hidePwsh` is false | | `hideOfficialTools` | true | Hide the official file tools this mode replaces; disabled entirely when `global.hideOfficialTools` is false | | `dynamicTools` | true | Publish the progressive SDK loader `ops_load_sdk`; effective only while `global.dynamicTools` is also on (AND gate) | +| `disclosurePolicy` | `on-demand` | YAML-only explicit first-round disclosure tier: `on-demand` keeps the legacy progressive path (resident + loader); `full` publishes the whole `allowedTools` pool in the first round with no loader, once the D3 publication slice lands. Effective only while `global.dynamicTools` is on — the explicit opt-in never overrides the bundle switch; while it is off the row keeps its legacy resident/always-loaded surface and the saved choice is preserved. Rejected on non-PTC rows; illegal values are errors naming the key and the received value or type. No upgrade, settings save or switch flip ever writes this key for you | | `batchingGuidance` | true for `ops-ptc`, false otherwise | Teach once at session start (logged history, system prompt untouched), then correct the shape of root `run_code` programs: `bypass`, `plan` and `shape` each latch once per turn and re-arm when the program complies; the notice rides the triggering result as `additionalContexts` and never blocks or rewrites the program. Text lives in the editable `prompts/batching-guidance.md`. YAML-only; an omitted `ops-ptc` section in a legacy whole-row profile override retains true; explicit false disables it | | `spillRetentionDays` | 30 | Host subprocess spill retention window; `0` reports only and never deletes | | `spillTotalBudgetMiB` | 256 | Total retained spill budget in MiB; `0` disables the budget | @@ -89,7 +90,9 @@ The exact first-round `ops_*` name sets for the five legacy inputs — defaults, `global.dynamicTools:false`, row `dynamicTools:false`, that row plus `alwaysLoaded:[stat]`, and `residentTools:false` — are locked letter by letter in the [D1 truth table](design-decisions-20261119.md). Old `dynamicTools:false` publishes only the -resident/always-loaded surface and must not be read as a full selection. +resident/always-loaded surface and must not be read as a full selection; the explicit full tier +is a separate `disclosurePolicy` opt-in whose parsing contract and conservative precedence are +recorded in the same document's D2 addendum, with publication owned by the D3 slice. ### Batch-call guidance (`batchingGuidance`) diff --git a/docs/design-decisions-20261119.md b/docs/design-decisions-20261119.md index 4e5893a3..0805dec4 100644 --- a/docs/design-decisions-20261119.md +++ b/docs/design-decisions-20261119.md @@ -169,3 +169,48 @@ this slice — that is what the suite asserts. 1–5-per-call cap with the count-cap-free resident startup list, and the legal custom PTC integration misdiagnosed `PUBLICATION_UNAVAILABLE: ops_result`. The suite now reports 13 passed / 0 skipped / 0 failed on the baseline. + +## D2 addendum — the explicit disclosure policy key, parsed (2026-11-19) + +Card [#986](https://github.com/T-Auto/dsh-ops/issues/986), on top of this table's +branch. This addendum records what D2 locked and resolves sign-off item 2's +precedence question conservatively; publication of the full tier remains D3. + +**The key.** The `ptc` and `ops-ptc` rows accept a YAML-only +`disclosurePolicy: 'on-demand' | 'full'` (default `'on-demand'` = the legacy path +of this table, letter for letter). Illegal values are errors naming the key and +the received value or type; non-PTC rows (`minimal`, `standard`, `cordis`) +reject the key with `disclosurePolicy applies to the ptc and ops-ptc rows only`. +The builtin consumer plan slice and custom integrations keep the legacy default +until D3/D4 teach them the tier. + +**Sign-off item 2, resolved conservatively per #979 §4.** An explicit full +selection is a *distinct opt-in* and never overrides the bundle switch: while +`global.dynamicTools` is off, the full tier is not applied — the row keeps its +legacy resident/alwaysLoaded surface (row 2 of this table) — and the saved +choice stays untouched in the row config, re-applying when the switch returns +on. This keeps every §4 guarantee in one direction: upgrades, settings saves +and switch flips never write or erase the key, and the operator's off stays +absolute for every disclosure shape. The row-level `dynamicTools` keeps its +loader-only meaning: the full tier publishes no loader (#979 §2), so a row that +also turns the loader off keeps its explicit tier. `effectiveMode` encodes this +gate; the saved intent is preserved exactly like the existing dynamicTools AND +gate preserves it. + +**Serialization.** `hostConfig` (the save-path serializer) preserves an explicit +choice verbatim and never auto-writes the key: reading → saving an unrelated +setting → reading again keeps `'full'`, and every legacy input — the five rows +of this table included — resolves with the key absent at both the raw and +serialized layers. + +**Still not implemented.** Truth-table row T stays a target: nothing publishes +the full tier yet (D3 owns `src/publication/ptc.ts` and the builtin plan slice), +there is no settings-page entry (D5), and sign-off items 1, 3 and 4 remain open +as recorded above. + +**Verification.** `npm run build`, then `node test/run.mjs ptc-disclosure-policy` +— 8 checks, red first (8 of 8 failed on the unimplemented baseline, the key +rejected as unknown), green after. `node test/run.mjs ptc-config-truth` 13/0/0, +`node test/run.mjs resident-switch` 24/0/0 and `node test/run.mjs settings-rows` +14/0/0 unchanged on the new parsing path, worker +`DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker` (sha256 2b5f988d). diff --git "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" index d28fd717..f61d0d02 100644 --- "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" +++ "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" @@ -149,3 +149,36 @@ PTC 行实际为该入口发布的是类型化 loader `ops_load_sdk`(真值表 - 补录审计(同日追加):新增两项检查——loader 单次 1–5 上限与常驻启动名单无数量 上限、合法自定义 PTC integration 被误诊 `PUBLICATION_UNAVAILABLE: ops_result`。 套件在基准上现为 13 通过 / 0 跳过 / 0 失败。 + +## D2 附录——显式披露策略键已解析(2026-11-19) + +任务卡 [#986](https://github.com/T-Auto/dsh-ops/issues/986),叠在本真值表分支之上。本附录 +记录 D2 锁定的内容,并按保守取向解决签字决定项 2 的优先级问题;全量档的发布仍归 D3。 + +**键。** `ptc` 与 `ops-ptc` 行接受 YAML-only 的 +`disclosurePolicy: 'on-demand' | 'full'`(缺省 `'on-demand'` = 本表的旧路径,逐字一致)。 +非法值报错点名键与收到的值或类型;非 PTC 行(`minimal`、`standard`、`cordis`)以 +`disclosurePolicy applies to the ptc and ops-ptc rows only` 拒绝该键。内置消费者计划切片 +与自定义 integration 在 D3/D4 学会该档位之前保持旧缺省。 + +**签字决定项 2,按 #979 §4 取保守解。** 显式全量选择是**独立的 opt-in**,绝不压过总 +开关:`global.dynamicTools` 关闭期间全量档不生效——该行保持本表第 2 行的旧 +resident/alwaysLoaded 面——而已保存的选择原样留在行配置里,开关回开后重新生效。这使 +§4 的每条保证都成立:升级、保存设置与开关翻转从不写入或抹掉该键,操作者的关闭对 +一切披露形态保持绝对。行级 `dynamicTools` 保持只管 loader 的含义:全量档不发布 +loader(#979 §2),因此同时关掉 loader 的行保留自己的显式档位。该门编码在 +`effectiveMode`;保存的意图与现有 dynamicTools 与门一样原样保留。 + +**序列化。** `hostConfig`(保存路径的序列化器)逐字保留显式选择,且从不自动写键: +读取 → 保存一项无关设置 → 再读取,`'full'` 不丢;包括本表五行在内的每个旧输入在 +raw 与序列化两层都保持该键缺席。 + +**仍未实现。** 真值表 T 行仍是目标:尚无任何路径发布全量档(D3 负责 +`src/publication/ptc.ts` 与内置计划切片),没有设置页入口(D5),签字决定项 1、3、4 +维持上文原状。 + +**验证。** `npm run build` 后 `node test/run.mjs ptc-disclosure-policy`——8 项检查, +先红(未实现基线上 8/8 失败,键被拒绝为 unknown),实现后全绿。 +`node test/run.mjs ptc-config-truth` 13/0/0、`node test/run.mjs resident-switch` +24/0/0、`node test/run.mjs settings-rows` 14/0/0 在新解析路径上不变,worker 用 +`DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker`(sha256 2b5f988d)。 diff --git "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" index cc08a167..5a3835a8 100644 --- "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" +++ "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" @@ -47,6 +47,7 @@ OPS 沿用单行插件页:bundle 只插入一个 `dsh-ops` 行。内置预设 | `hidePwsh` | true | Bash 可用时隐藏 Windows 官方 shell;`global.hidePwsh` 为 false 时整体失效 | | `hideOfficialTools` | true | 隐藏本模式替代的官方文件工具;`global.hideOfficialTools` 为 false 时整体失效 | | `dynamicTools` | true | 发布渐进式 SDK 加载器 `ops_load_sdk`;与 `global.dynamicTools` 同时开启才生效(与门) | +| `disclosurePolicy` | `on-demand` | YAML-only 显式首轮披露档位:`on-demand` 保持旧渐进路径(常驻 + loader);`full` 在 D3 发布切片落地后首轮直接发布整个 `allowedTools` 池且无 loader。仅在 `global.dynamicTools` 开启时生效——显式 opt-in 不压过总开关;关闭期间该行保持旧 resident/alwaysLoaded 面,已保存的选择原样保留。非 PTC 行拒绝该键;非法值报错点名键与收到的值或类型。升级、保存设置与开关翻转都不会替你写入该键 | | `batchingGuidance` | `ops-ptc` 为 true,其它模式为 false | 会话首轮注入一次完整教学(进入历史、不改系统提示词),之后按根 `run_code` 程序形状纠偏:`bypass`/`plan`/`shape` 三类每类每轮各锁一次、合规即解锁;提醒以 `additionalContexts` 附在触发调用结果上,不阻断也不改写程序。文案来自可编辑的 `prompts/batching-guidance.md`。仅 YAML 可配;旧 profile 整行覆盖配置且缺少 `ops-ptc` 节时仍默认 true,显式 false 可关闭 | | `spillRetentionDays` | 30 | 宿主 subprocess spill 保留窗口;`0` 只报告、永不删除 | | `spillTotalBudgetMiB` | 256 | 保留总字节预算(MiB);`0` 关闭预算 | @@ -61,7 +62,8 @@ OPS 沿用单行插件页:bundle 只插入一个 `dsh-ops` 行。内置预设 五种旧配置输入——缺省、`global.dynamicTools:false`、行级 `dynamicTools:false`、行关闭加 `alwaysLoaded:[stat]` 与 `residentTools:false`——的首轮 `ops_*` 名字集合已在 [D1 真值表](设计决策-20261119.md) 中逐字锁定。旧 `dynamicTools:false` 只发布 -resident/alwaysLoaded 面,不得解读为全量选择。 +resident/alwaysLoaded 面,不得解读为全量选择;显式全量是独立的 `disclosurePolicy` +opt-in,其解析契约与保守优先级记录在同一文档的 D2 附录,发布归 D3 切片。 ### 批量调用提示(`batchingGuidance`) diff --git a/src/presets/config.ts b/src/presets/config.ts index 5fca3fa0..927097b6 100644 --- a/src/presets/config.ts +++ b/src/presets/config.ts @@ -6,7 +6,7 @@ * and a malformed row still fails loud and names the offending key. */ import { LEGACY_FASTCTX_KEYS, resolveConfig, type ResolvedConfig } from '../config.js' -import { parsePtcConfig, type PtcConfig } from '../ptc/config.js' +import { parsePtcConfig, type DisclosurePolicy, type PtcConfig } from '../ptc/config.js' import { editable, settingsConfig, type Field } from '../settings/schema.js' import { configObject } from '../settings/schema.js' import type { ModeId } from './types.js' @@ -15,6 +15,8 @@ import type { ModeId } from './types.js' export interface ModeConfig extends ResolvedConfig { allowedTools: PtcConfig['allowedTools'] alwaysLoaded: PtcConfig['alwaysLoaded'] + /** Explicit first-round disclosure tier; the default 'on-demand' is the legacy path (#986). */ + disclosurePolicy: DisclosurePolicy /** Publish Bash for this preset's sessions. */ bash: boolean /** Hide the official Windows shell by default while Bash is available. */ @@ -37,12 +39,14 @@ export const MODE_CONFIG_KEYS: readonly string[] = Object.freeze([ 'binaryPath', 'required', 'denyCredentialReads', 'promptPolicy', 'extraGuidance', 'bashPath', 'publishBashTool', 'allowSystemShellFallback', 'configUpdateVersion', 'spillRetentionDays', 'spillTotalBudgetMiB', 'spillSweepEnabled', - 'allowedTools', 'alwaysLoaded', + 'allowedTools', 'alwaysLoaded', 'disclosurePolicy', 'bash', 'hidePwsh', 'hideOfficialTools', 'dynamicTools', 'residentTools', 'batchingGuidance', 'batchingTelemetry', ]) -const PTC_KEYS = new Set(['allowedTools', 'alwaysLoaded']) +const PTC_KEYS = new Set(['allowedTools', 'alwaysLoaded', 'disclosurePolicy']) +/** The mode rows that own a PTC publication surface, hence the disclosure policy. */ +const PTC_POLICY_MODES: readonly ModeId[] = ['ptc', 'ops-ptc'] /** Boolean switches and gates, each supplying its own default below. */ const SWITCH_KEYS = new Set(['bash', 'hidePwsh', 'hideOfficialTools', 'dynamicTools', 'residentTools', 'batchingGuidance', 'batchingTelemetry']) @@ -63,6 +67,11 @@ export function resolveModeConfig(rawConfig?: unknown, mode?: ModeId): ModeConfi throw new Error(`unknown key(s) ${JSON.stringify(key)}; accepted keys are ${MODE_CONFIG_KEYS.map(entry => JSON.stringify(entry)).join(', ')}`) } } + // The disclosure policy names a PTC publication tier: a row without a PTC + // surface must fail loud instead of silently carrying the key. + if (Object.hasOwn(raw, 'disclosurePolicy') && mode !== undefined && !PTC_POLICY_MODES.includes(mode)) { + throw new Error('disclosurePolicy applies to the ptc and ops-ptc rows only') + } const core = Object.fromEntries(Object.entries(raw).filter(([key]) => !PTC_KEYS.has(key) && !SWITCH_KEYS.has(key))) const ptc = Object.fromEntries(Object.entries(raw).filter(([key]) => PTC_KEYS.has(key))) const switches = Object.fromEntries(Object.entries(raw).filter(([key]) => SWITCH_KEYS.has(key))) @@ -76,6 +85,7 @@ export function resolveModeConfig(rawConfig?: unknown, mode?: ModeId): ModeConfi const resolved = resolveConfig({ ...core, publishBashTool: bash }) const api = parsePtcConfig(ptc) return { ...resolved, allowedTools: api.allowedTools, alwaysLoaded: api.alwaysLoaded, + disclosurePolicy: api.disclosurePolicy ?? 'on-demand', bash, hidePwsh: sw('hidePwsh', true), hideOfficialTools: sw('hideOfficialTools', true), dynamicTools: sw('dynamicTools', true), residentTools: mode !== 'minimal' && sw('residentTools', true), diff --git a/src/ptc/config.ts b/src/ptc/config.ts index 1d255633..cde03516 100644 --- a/src/ptc/config.ts +++ b/src/ptc/config.ts @@ -4,13 +4,24 @@ import type { FileToolName, PtcToolName } from '../capabilities/registry.js' import { migrateRawNames } from '../compat/alias.js' export type ApiName = FileToolName | PtcToolName export const PTC_LOADABLE_TOOLS: readonly ApiName[] = [...FILE_TOOLS, ...PTC_TOOLS].filter(tool => !PTC_RESIDENT_TOOLS.includes(tool)) -export interface PtcConfig { binaryPath?: string; required: boolean; allowedTools: ApiName[]; alwaysLoaded: ApiName[] } +/** The explicit first-round disclosure tier of a PTC row (#986/#979). */ +export type DisclosurePolicy = 'on-demand' | 'full' +export interface PtcConfig { binaryPath?: string; required: boolean; allowedTools: ApiName[]; alwaysLoaded: ApiName[]; disclosurePolicy?: DisclosurePolicy } +// parsePtcConfig always fills disclosurePolicy; consumer plan slices that have +// not been taught the tier yet (the builtin plan until D3, custom integrations +// until D4) may omit it and keep the legacy on-demand default. export function parsePtcConfig(raw: unknown = {}): PtcConfig { if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error('PTC config must be an object') const input = raw as Record - for (const key of Object.keys(input)) if (!['binaryPath', 'required', 'allowedTools', 'alwaysLoaded'].includes(key)) throw new Error('Unknown PTC config key: ' + key) + for (const key of Object.keys(input)) if (!['binaryPath', 'required', 'allowedTools', 'alwaysLoaded', 'disclosurePolicy'].includes(key)) throw new Error('Unknown PTC config key: ' + key) if (input.binaryPath !== undefined && typeof input.binaryPath !== 'string') throw new Error('binaryPath must be a string') if (input.required !== undefined && typeof input.required !== 'boolean') throw new Error('required must be boolean') + // The default 'on-demand' is the legacy path; 'full' is a distinct explicit + // opt-in (#979 §4) — no upgrade, save or switch may pick it for the user. + const policy = input.disclosurePolicy + if (policy !== undefined && (typeof policy !== 'string' || (policy !== 'on-demand' && policy !== 'full'))) { + throw new Error('disclosurePolicy must be "on-demand" or "full"; got ' + (JSON.stringify(policy) ?? String(policy)) + ' (' + typeof policy + ')') + } const names = (key: string, fallback: readonly ApiName[]): ApiName[] => { const value = input[key] ?? fallback if (!Array.isArray(value)) throw new Error(key + ' must be an array of raw API names') @@ -19,5 +30,6 @@ export function parsePtcConfig(raw: unknown = {}): PtcConfig { return [...new Set(migrated as ApiName[])] } return { binaryPath: input.binaryPath, required: input.required ?? false, - allowedTools: names('allowedTools', PTC_LOADABLE_TOOLS), alwaysLoaded: names('alwaysLoaded', []) } + allowedTools: names('allowedTools', PTC_LOADABLE_TOOLS), alwaysLoaded: names('alwaysLoaded', []), + disclosurePolicy: policy === 'full' ? 'full' : 'on-demand' } } diff --git a/src/settings/merged.ts b/src/settings/merged.ts index 4818859f..8d0f5102 100644 --- a/src/settings/merged.ts +++ b/src/settings/merged.ts @@ -232,6 +232,12 @@ export function effectiveMode(state: FeatureState, global: GlobalSwi hidePwsh: flag(raw, 'hidePwsh', state.config.hidePwsh) && global.hidePwsh && !pwsh7Link, hideOfficialTools: flag(raw, 'hideOfficialTools', true) && global.hideOfficialTools, dynamicTools: flag(raw, 'dynamicTools', true) && global.dynamicTools, + // Sign-off item 2 (docs/design-decisions-20261119.md), taken conservatively + // per #979 §4: the explicit full tier is its own opt-in and never overrides + // the bundle switch. While the switch is off the effective slice says + // on-demand, so the row keeps its legacy resident/alwaysLoaded surface; the + // saved choice stays untouched in state.raw and re-applies when switched on. + ...(raw.disclosurePolicy === 'full' && !global.dynamicTools ? { disclosurePolicy: 'on-demand' } : {}), } } diff --git a/test/ptc-disclosure-policy.test.mjs b/test/ptc-disclosure-policy.test.mjs new file mode 100644 index 00000000..a1d86c05 --- /dev/null +++ b/test/ptc-disclosure-policy.test.mjs @@ -0,0 +1,142 @@ +/** The D2 parsing contract: one explicit disclosure policy key on the PTC rows. + * + * #979 splits the two-tier OPS-PTC work into slices; this suite locks the D2 + * slice (#986): the row key `disclosurePolicy` ('on-demand' | 'full', default + * 'on-demand' = the legacy path), its validation diagnostics, its survival + * through the settings save path, and the conservative precedence taken for the + * D1 sign-off item 2 — the explicit full tier never overrides the bundle + * `dynamicTools` switch, and no save, upgrade or switch flip ever writes the + * key for the user (#979 §4). Publication of the full tier itself is D3; every + * assertion here is resolver-level only. + */ +import { assert, test, report } from './lib/harness.mjs' +import { parsePtcConfig } from '../lib/ptc/config.js' +import { modeConfigForm, modeFields, resolveModeConfig } from '../lib/modes/config.js' +import { Config, effectiveMode, hostConfig, resolveMergedConfig } from '../lib/settings/merged.js' +import { changedFields, readRowValues } from '../lib/client/row-schema.js' + +await test('input class 1 (key absent): every surface keeps the legacy default and slice', () => { + // The parsed contract gains a defaulted field; the effective slice and every + // published surface stay letter-identical to the pre-D2 baseline. + assert.equal(parsePtcConfig({}).disclosurePolicy, 'on-demand') + for (const mode of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) { + assert.equal(resolveModeConfig({}, mode).disclosurePolicy, 'on-demand', mode) + } +}) + +await test('input class 2 (legal value): on-demand and full parse and ride the settings path', () => { + assert.equal(parsePtcConfig({ disclosurePolicy: 'on-demand' }).disclosurePolicy, 'on-demand') + assert.equal(parsePtcConfig({ disclosurePolicy: 'full' }).disclosurePolicy, 'full') + for (const mode of ['ptc', 'ops-ptc']) { + assert.equal(resolveModeConfig({ disclosurePolicy: 'full' }, mode).disclosurePolicy, 'full', mode) + assert.equal(resolveModeConfig({ disclosurePolicy: 'on-demand' }, mode).disclosurePolicy, 'on-demand', mode) + } + // D2 adds no settings-page field: the key stays YAML-only (the entry is D5). + assert.equal('disclosurePolicy' in modeFields('ops-ptc'), false) + assert.equal('disclosurePolicy' in Config.dict['ops-ptc'].dict, false) +}) + +await test('input class 3 (misspelled value): the diagnostic names the key and the value', () => { + assert.throws(() => parsePtcConfig({ disclosurePolicy: 'ondemand' }), error => { + assert.match(error.message, /disclosurePolicy must be "on-demand" or "full"/) + assert.match(error.message, /"ondemand"/) + return true + }) +}) + +await test('input class 4 (wrong type): the diagnostic names the key and the type', () => { + for (const value of [true, 5, ['full']]) { + assert.throws(() => parsePtcConfig({ disclosurePolicy: value }), error => { + assert.match(error.message, /disclosurePolicy must be "on-demand" or "full"/) + assert.match(error.message, new RegExp('\\b' + typeof value + '\\b')) + return true + }, JSON.stringify(value)) + } +}) + +await test('modes without a PTC surface reject the key with a readable diagnostic', () => { + // The key names a PTC publication tier; a Native row carrying it is a config + // mistake that must fail loud, not silently drift (the denyCredentialReads + // precedent for cross-surface keys). + for (const mode of ['minimal', 'standard', 'cordis']) { + assert.throws(() => resolveModeConfig({ disclosurePolicy: 'full' }, mode), + /disclosurePolicy applies to the ptc and ops-ptc rows only/, mode) + assert.throws(() => resolveModeConfig({ disclosurePolicy: 'full' }, mode), /disclosurePolicy/, mode) + } +}) + +await test('round trip: read → save an unrelated setting → read again keeps the explicit choice', () => { + const raw = { 'ops-ptc': { disclosurePolicy: 'full', allowedTools: ['grep', 'stat'] } } + const first = resolveMergedConfig(raw) + assert.equal(first['ops-ptc'].config.disclosurePolicy, 'full') + assert.equal(first['ops-ptc'].raw?.disclosurePolicy, 'full') + // The host settings layer serializes through hostConfig and validates through + // the row descriptor; both preserve the key. + assert.equal(hostConfig(raw)['ops-ptc'].disclosurePolicy, 'full') + assert.equal(Config['~standard'].validate(raw).value['ops-ptc'].disclosurePolicy, 'full') + // The component's own settings descriptor keeps raw ordinary keys too. + const form = modeConfigForm('ops-ptc') + assert.equal(form['~standard'].validate({ disclosurePolicy: 'full' }).value.disclosurePolicy, 'full') + // The page writes only the field the operator touched: saving panel capture + // produces exactly one op and never touches the ops-ptc section. + const draft = readRowValues(raw) + draft['panel.enabled'] = true + assert.deepEqual(changedFields(draft, raw), [{ op: 'set', path: ['panel', 'enabled'], value: true }]) + // The host applies that one op; reading again still sees the explicit choice. + const after = resolveMergedConfig({ ...raw, panel: { enabled: true } }) + assert.equal(after['ops-ptc'].config.disclosurePolicy, 'full', 'the explicit selection is not lost') + assert.deepEqual(after['ops-ptc'].config.allowedTools, ['grep', 'stat']) +}) + +await test('legacy configs never gain the key; the global switch never rewrites it', () => { + // The D1 truth-table five inputs, each re-resolved on the new parsing path. + const inputs = [ + {}, + { global: { dynamicTools: false } }, + { 'ops-ptc': { dynamicTools: false } }, + { 'ops-ptc': { dynamicTools: false, alwaysLoaded: ['stat'] } }, + { 'ops-ptc': { residentTools: false } }, + ] + for (const raw of inputs) { + const merged = resolveMergedConfig(raw) + for (const mode of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) { + const slice = effectiveMode(merged[mode], merged.global) + assert.equal('disclosurePolicy' in slice, false, 'the effective slice stays letter-identical', mode) + assert.equal('disclosurePolicy' in (merged[mode].raw ?? {}), false, mode) + } + assert.equal(merged.ptc.config.disclosurePolicy, 'on-demand') + assert.equal(merged['ops-ptc'].config.disclosurePolicy, 'on-demand') + // hostConfig is the save-path serializer: it never auto-writes the key. + const saved = hostConfig(raw) + for (const mode of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) { + assert.equal('disclosurePolicy' in saved[mode], false, 'an upgrade or save never picks full for the user') + } + } +}) + +await test('the explicit full tier never overrides the bundle switch (sign-off item 2)', () => { + const saved = { 'ops-ptc': { disclosurePolicy: 'full' } } + const on = resolveMergedConfig(saved) + assert.equal(effectiveMode(on['ops-ptc'], on.global).disclosurePolicy, 'full') + assert.equal(resolveModeConfig(effectiveMode(on['ops-ptc'], on.global), 'ops-ptc').disclosurePolicy, 'full') + const off = resolveMergedConfig({ global: { dynamicTools: false }, ...saved }) + // The saved intent is kept; only the effective slice stops applying, exactly + // like the existing dynamicTools AND gate keeps the operator's intent. + assert.equal(off['ops-ptc'].config.disclosurePolicy, 'full') + assert.equal(off['ops-ptc'].raw?.disclosurePolicy, 'full') + assert.equal(hostConfig({ global: { dynamicTools: false }, ...saved })['ops-ptc'].disclosurePolicy, 'full') + assert.equal(effectiveMode(off['ops-ptc'], off.global).disclosurePolicy, 'on-demand', 'the bundle switch keeps its veto') + assert.equal(resolveModeConfig(effectiveMode(off['ops-ptc'], off.global), 'ops-ptc').disclosurePolicy, 'on-demand') + assert.equal(effectiveMode(off['ops-ptc'], off.global).dynamicTools, false) + // The row switch keeps its loader-only meaning; the full tier publishes no + // loader, so a row that also turns the loader off keeps its explicit tier. + const rowOff = resolveMergedConfig({ 'ops-ptc': { disclosurePolicy: 'full', dynamicTools: false } }) + assert.equal(effectiveMode(rowOff['ops-ptc'], rowOff.global).disclosurePolicy, 'full') + // Flipping the global switch alone never writes the key anywhere. + for (const global of [{ dynamicTools: false }, { dynamicTools: true }]) { + const flipped = resolveMergedConfig({ global }) + assert.equal('disclosurePolicy' in (flipped['ops-ptc'].raw ?? {}), false, JSON.stringify(global)) + } +}) + +report('ptc-disclosure-policy') From ec4d25c40b48bdddbf9fa30fa7a1e671597389f5 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:30:58 +0800 Subject: [PATCH 6/9] bench(tracks): script and judge the four OPS-PTC behavior tasks Four bench/tasks samples whose prompts state only the business goal: paged cursor continuation, catalog-error recovery, load-before-use and wait/multi-dispatch. A new pagedLog fixture family puts the target on page 2 and derives its answer from the bytes. bench/lib/tracks.mjs judges a scripted trajectory against each task's fixed facts (cursor advance in the next program on the same path, complete-page stop, advertised-set recovery, load order, revision-awaited edits) and names the fact that did not hold; verbatim re-issues after a delivered or input-refused envelope are no-progress retries while bounded-transient pauses count as real waits. The tracks CLI runs ten should-pass / should-fail trajectories and exits non-zero on any disagreement; the recovery rate counts only tracks that had a correction opportunity and says no denominator instead of 100%. bench-tracks pins both states red first (#993). --- bench/README.md | 20 +- bench/cli.mjs | 1 + bench/lib/fixtures/index.mjs | 8 +- bench/lib/fixtures/paged.mjs | 63 +++++ bench/lib/track-corpus.mjs | 209 +++++++++++++++ bench/lib/tracks.mjs | 299 ++++++++++++++++++++++ bench/tasks/ptc-catalog-recover.task.json | 12 + bench/tasks/ptc-paged-lookup.task.json | 12 + bench/tasks/ptc-sdk-load-order.task.json | 12 + bench/tasks/ptc-wait-dispatch.task.json | 15 ++ bench/tracks.mjs | 106 ++++++++ docs/CHANGELOG.md | 3 + test/bench-fixtures.test.mjs | 19 +- test/bench-tracks.test.mjs | 206 +++++++++++++++ 14 files changed, 977 insertions(+), 8 deletions(-) create mode 100644 bench/lib/fixtures/paged.mjs create mode 100644 bench/lib/track-corpus.mjs create mode 100644 bench/lib/tracks.mjs create mode 100644 bench/tasks/ptc-catalog-recover.task.json create mode 100644 bench/tasks/ptc-paged-lookup.task.json create mode 100644 bench/tasks/ptc-sdk-load-order.task.json create mode 100644 bench/tasks/ptc-wait-dispatch.task.json create mode 100644 bench/tracks.mjs create mode 100644 test/bench-tracks.test.mjs diff --git a/bench/README.md b/bench/README.md index e1aead58..8272638a 100644 --- a/bench/README.md +++ b/bench/README.md @@ -108,7 +108,7 @@ The `index-ab` command compares the walk with the opt-in `cache_dir` path on one ## 任务一览 -六个任务各自压一种能力,`ops_*` 工具名来自 `publicToolName()`。每个任务的期望值都由夹具生成器现算(`derive*`),判卷在 `bench/lib/verify.mjs`。 +十个任务各自压一种能力,`ops_*` 工具名来自 `publicToolName()`。前六个只判 `answer.txt` 这类最终工件;后四个是 V4 行为任务(#993),除最终工件外还由轨迹判定器验证达成方式(见下节)。每个任务的期望值都由夹具生成器现算(`derive*`),工作区判卷在 `bench/lib/verify.mjs`。 | 任务 | 工作量 | 主要测什么 | 相关工具 | | --- | --- | --- | --- | @@ -118,3 +118,21 @@ The `index-ab` command compares the walk with the opt-in `cache_dir` path on one | `csv-sum` | 8000 行 CSV,汇总 EU 区 `amount` 列 | 上下文控制:把聚合交给 shell 而不是把 8000 行读进上下文,`answer.txt` 只写整数和 | `ops_bash` 聚合;`ops_read_files` 只看表头确认列;`ops_write_file` 写答案 | | `gate-repair-loop` | 12 个 stage 模块(每个约 70 行)+ `check.js` | 迭代修复循环:门禁停在第一个坏 stage,每个 stage 只错一处绑定常量、契约写在模块自己的注释里,跑→修→再跑直到退 0;`check.js` 字节不变是 invariant | `ops_bash` 跑门禁;`ops_read_files` 读 stage;`ops_edit_file` 按契约修常量;`ops_write_file` 写 `answer.txt` | | `noisy-test-triage` | 2400 个测试、恰好一行 `not ok` | 精确提取:跑测试重定向到文件,在日志里只捞那一行失败测试名写入 `failure.txt`(必须精确匹配),且 runner 仍退 1 是 invariant | `ops_bash` 跑并重定向;`ops_grep` 只捞那一行(如 `^not ok`,可配 `outputMode: summary/content`);必要时 `ops_read_files`;`ops_write_file` 写结果 | +| `ptc-paged-lookup` | 600 行单文件 `logs/stream.log`,目标行在第 2 页 | 分页游标续读:partial 带 `nextOffset` 后必须在下一程序同路径续读,末页 complete 即停;原样重读、提前收尾都判 FAIL | `ops_inspect_local_file` 续读;`ops_write_file` 写答案 | +| `ptc-catalog-recover` | grepTree 语料,5 个文件含 `NEEDLE_CATALOG` | 目录错误纠偏:叫错工具名被拒后,下一步必须用当时可用集合里的替代工具,不许再追不存在的名字 | `ops_grep`(或 `ops_glob`/`ops_inspect_local_file`);`ops_write_file` 写答案 | +| `ptc-sdk-load-order` | grepTree 语料,3 个文件含 `NEEDLE_SDKLOAD` | 未加载工具:识别需先 `ops_load_sdk`,下一程序才可调;原样重试不加载判 FAIL | `ops_load_sdk` 加载 `grep`;下一程序 `ops_grep`;`ops_write_file` 写答案 | +| `ptc-wait-dispatch` | configTree 三层配置 + 8 个包级诱饵 | 等待/多派发:层文件并行读取一次派发、编辑要等待读取返回的 revision;锁忙(`bounded-transient`)后的原样重发是合理等待不是无进展重试;正常完成无须重试的反例必须保留 | `ops_inspect_local_file` 并读;`ops_edit_file` 带期望 revision;`ops_write_file` 写答案 | + +## 行为轨迹判定 / Behavior tracks + +`node bench/cli.mjs tracks`(V4,#993)不调用模型、不派生 worker:它把 `bench/lib/track-corpus.mjs` 里的十段脚本化轨迹(四个 V4 任务各配应过与应败)逐条交给 `bench/lib/tracks.mjs` 的判定器,应过轨迹必须判 PASS、应败轨迹必须判 FAIL,且失败输出指明是哪项事实不符;任何一条判定与声明不符,命令退 1。 + +恢复率只统计真的出现过纠偏机会(被拒 / 出错 / 冲突 / 带 `nextOffset` 的 partial)的轨迹:全量语料报 `4/9`;选中的轨迹全无纠偏机会时输出「no denominator」而不是 100%(可用 `--track ptc-wait-immediate-pass` 复现)。 + +复跑:`node bench/cli.mjs tracks`;按任务收窄 `--task `;按轨迹收窄 `--track `。红绿两步如实记录:先写 `test/bench-tracks.test.mjs` 断言并跑红(判定器与语料缺失,`ERR_MODULE_NOT_FOUND`),实现落地后 `node test/run.mjs bench-tracks` 全绿。V5 会用同一套判定器接真实模型的轨迹。 + +The `tracks` command (V4, #993) sends no model request and spawns no worker: it feeds the ten scripted trajectories in `bench/lib/track-corpus.mjs` (each V4 task carries a should-pass and a should-fail track) to the verifier in `bench/lib/tracks.mjs`; a should-pass track must judge PASS, a should-fail one must judge FAIL naming the fact that did not hold, and any disagreement exits 1. + +The recovery rate counts only trajectories that actually had a correction opportunity (a rejection, an error, a conflict, or a partial page carrying `nextOffset`): the full corpus reports `4/9`; a selection with no opportunity at all prints an explicit no-denominator line instead of 100% (reproduce with `--track ptc-wait-immediate-pass`). + +Rerun with `node bench/cli.mjs tracks`, narrow with `--task ` or `--track `. The red step was recorded honestly first — the suite failed at the import because no verifier existed — and `node test/run.mjs bench-tracks` is green now. V5 feeds real captured trajectories through the same verifiers. diff --git a/bench/cli.mjs b/bench/cli.mjs index 4be1de20..e8ca71ca 100644 --- a/bench/cli.mjs +++ b/bench/cli.mjs @@ -34,6 +34,7 @@ export const COMMANDS = [ { name: 'tools', slice: 'P13', target: 'bench/tools/index.mjs', summary: 'model-free deterministic tool-level comparison' }, { name: 'calls', slice: 'P14', target: 'bench/calls/index.mjs', summary: 'host-level timing over a real corpus: official versus OPS, or two OPS builds' }, { name: 'index-ab', slice: 'G4 (#614)', target: 'bench/index-ab/index.mjs', summary: 'index versus walk A/B on one corpus with the same worker binary' }, + { name: 'tracks', slice: 'V4 (#993)', target: 'bench/tracks.mjs', summary: 'judge the scripted OPS-PTC behavior tracks: paging, catalog, load order, wait' }, ] const AB_FLAGS_WITH_VALUE = ['--task', '--repeat', '--folder', '--base', '--cand'] diff --git a/bench/lib/fixtures/index.mjs b/bench/lib/fixtures/index.mjs index db383c1f..d2f362e6 100644 --- a/bench/lib/fixtures/index.mjs +++ b/bench/lib/fixtures/index.mjs @@ -3,8 +3,8 @@ * name -> builder registry, and the derivation functions that recompute each * builder's expectation from the bytes it wrote. * - * The catalogue is split by family — `csv` / `log` / `tree` / `symbol` / `corpus` - * / `deep` / `repair` — but re-exported from this one path so the tasks, `run.mjs` + * The catalogue is split by family — `csv` / `log` / `tree` / `symbol` / `paged` + * / `corpus` / `deep` / `repair` — but re-exported from this one path so the tasks * and the suites keep a single import. The scaffold itself lives in * `./internal.mjs`, which imports nothing, so the graph is a DAG: barrel -> * family -> scaffold. The shared names are still re-exported from here, because @@ -20,6 +20,7 @@ import { symbolTree } from './symbol.mjs' import { deepModule } from './deep.mjs' import { deepCorpus } from './corpus.mjs' import { brokenProject, gatePipeline } from './repair.mjs' +import { pagedLog } from './paged.mjs' export { deriveLargeCsv, largeCsv } from './csv.mjs' export { deriveLogStats, deriveNoisySuite, logStats, noisySuite } from './log.mjs' @@ -28,6 +29,7 @@ export { deriveSymbolTree, symbolTree } from './symbol.mjs' export { deepModule, deriveDeepModule } from './deep.mjs' export { deepCorpus } from './corpus.mjs' export { brokenProject, gatePipeline } from './repair.mjs' +export { derivePagedLog, pagedLog } from './paged.mjs' export { drawIndices, hexRun, mulberry32, pad, pick, randInt, WORDS } from './internal.mjs' /** @@ -58,5 +60,5 @@ export function generate(name, params = {}) { */ export const GENERATORS = { grepTree, renameTree, largeCsv, brokenProject, logStats, configTree, - deepModule, deepCorpus, noisySuite, symbolTree, gatePipeline, + deepModule, deepCorpus, noisySuite, symbolTree, gatePipeline, pagedLog, } diff --git a/bench/lib/fixtures/paged.mjs b/bench/lib/fixtures/paged.mjs new file mode 100644 index 00000000..56734c0b --- /dev/null +++ b/bench/lib/fixtures/paged.mjs @@ -0,0 +1,63 @@ +/** + * The `paged` fixture family: one long log file whose answer line sits beyond the + * first page, for the V4 paged-cursor behavior task (#993). + * + * The shape is the point, not the content: a 600-line file read at the default + * 200-line page size puts the marked line on page 2, so a trajectory must continue + * at the envelope's nextOffset instead of re-reading page 1 or stopping early. + * The marker token appears exactly once and every other payload is lowercase words + * plus lowercase hex, so a wrong guess cannot pass and the exported answer is + * recomputable from the bytes alone. + * + * @module dsh-ops/bench/fixtures/paged + */ + +import { hexRun, mulberry32, pad, pick, WORDS } from './internal.mjs' + +/** + * `pagedLog` — one log file with exactly one marked line at a chosen position. + * @param {{seed?: number, lines?: number, markerAt?: number}} params - the shape of the log. + * @returns {{files: {path: string, content: string}[], exports: {answer: string, markerLine: number, markerText: string, totalLines: number}}} the fixture. + */ +export function pagedLog(params = {}) { + const seed = params.seed ?? 41 + const lines = Math.max(120, params.lines ?? 600) + const markerAt = params.markerAt ?? 350 + if (!Number.isInteger(markerAt) || markerAt < 2 || markerAt > lines) { + throw new RangeError('markerAt must be an integer line index between 2 and lines') + } + const next = mulberry32(seed) + const levels = ['INFO', 'INFO', 'DEBUG', 'INFO', 'WARN'] + const rows = [] + const stamp = (index) => '2025-03-' + pad((index % 28) + 1, 2) + 'T' + pad(index % 24, 2) + ':' + pad(index % 60, 2) + ':' + pad((index * 7) % 60, 2) + 'Z' + for (let index = 1; index <= lines; index += 1) { + const payload = pick(next, WORDS) + ' ' + pick(next, WORDS) + ' ' + hexRun(next, 8).toLowerCase() + rows.push(pick(next, levels) + '|' + stamp(index) + '|' + payload) + } + const markerText = 'TARGET-' + hexRun(next, 8).toUpperCase() + ' ' + pick(next, WORDS) + ' ' + pick(next, WORDS) + rows[markerAt - 1] = 'WARN|' + stamp(markerAt) + '|' + markerText + const content = rows.join('\n') + '\n' + // The token must be unique in the bytes, or the exported answer would be a lie. + if (content.split('TARGET-').length !== 2) throw new Error('the TARGET marker must appear exactly once in the log') + return { + files: [{ path: 'logs/stream.log', content }], + exports: { answer: markerAt + ' ' + markerText, markerLine: markerAt, markerText, totalLines: lines }, + } +} + +/** + * Recompute `pagedLog`'s exports from the generated bytes. + * @param {{path: string, content: string}[]} files - the generated tree. + * @returns {object} the recomputed expectation. + * @throws {Error} when the log file is missing or the marker is not unique. + */ +export function derivePagedLog(files) { + const file = files.find((entry) => entry.path === 'logs/stream.log') + if (file === undefined) throw new Error('derivePagedLog: logs/stream.log is missing') + const rows = file.content.replace(/\n$/, '').split('\n') + const marked = rows.map((row, index) => (row.includes('TARGET-') ? index : -1)).filter((index) => index >= 0) + if (marked.length !== 1) throw new Error('derivePagedLog: the TARGET marker must appear exactly once, found ' + marked.length) + const markerLine = marked[0] + 1 + const markerText = rows[marked[0]].split('|')[2] + return { answer: markerLine + ' ' + markerText, markerLine, markerText, totalLines: rows.length } +} diff --git a/bench/lib/track-corpus.mjs b/bench/lib/track-corpus.mjs new file mode 100644 index 00000000..2d8f25ef --- /dev/null +++ b/bench/lib/track-corpus.mjs @@ -0,0 +1,209 @@ +/** + * The scripted V4 behavior corpus (#993): the fixed task descriptors and the + * should-pass / should-fail trajectories that prove the verifier can tell them + * apart before any real model ever runs (V5 owns real trajectories). + * + * Every track derives its expected values from the real fixture bytes — the + * correct answer, the match lists, the winner path — so a scripted pass can never + * drift from the workspace the sweep would generate. The prompts in the task files + * stay business-only; nothing here leaks the judged facts into them. + * + * Envelope vocabulary follows the shipped outcome shapes (status / error.code / + * error.retry). UNKNOWN_API and NOT_IN_SDK are corpus markers for the feedback the + * real SDK signals by absence; V5 replaces them with captured envelopes. + * + * @module dsh-ops/bench/track-corpus + */ + +import { GENERATORS, generate } from './fixtures/index.mjs' +import { loadTasks, TASKS_DIR } from './tasks.mjs' + +/** The four V4 behavior tasks, in corpus order. */ +export const TRACK_TASK_IDS = ['ptc-paged-lookup', 'ptc-catalog-recover', 'ptc-sdk-load-order', 'ptc-wait-dispatch'] + +/** + * Load the four task descriptors with their fixtures generated and their facts fixed. + * @param {{tasksDir?: string}} [options] - override the corpus directory (suites pass a temp copy). + * @returns {object[]} one descriptor per task: {id, task, files, exports, sdk, budget, answer, verify}. + * @throws {Error} when a task file is missing, fails validation, or its fixture cannot generate. + */ +export function loadTrackTasks(options = {}) { + const dir = options.tasksDir ?? TASKS_DIR + const corpus = loadTasks(dir, { generators: GENERATORS }) + return TRACK_TASK_IDS.map((id) => { + const task = corpus.find((entry) => entry.id === id) + if (task === undefined) throw new Error('track task "' + id + '" is missing from ' + dir) + return buildTaskEntry(task) + }) +} + +/** Build one task descriptor: the fixture bytes, the fixed permissions, budget, answer and verifier facts. */ +function buildTaskEntry(task) { + const { files, exports } = generate(task.fixture.generator, task.fixture.params ?? {}) + const base = { id: task.id, task, files, exports, answer: { path: 'answer.txt', content: '' }, budget: { maxPrograms: 8 }, sdk: [], verify: {} } + if (task.id === 'ptc-paged-lookup') { + return { ...base, + sdk: ['ops_inspect_local_file', 'ops_load_sdk', 'ops_write_file'], + answer: { path: 'answer.txt', content: String(exports.answer) }, + verify: { path: 'logs/stream.log', pageSize: 200 }, + } + } + if (task.id === 'ptc-catalog-recover') { + return { ...base, + sdk: ['ops_inspect_local_file', 'ops_grep', 'ops_glob', 'ops_write_file'], + answer: { path: 'answer.txt', content: exports.matches.join('\n') }, + verify: { needle: task.fixture.params.needle, count: task.fixture.params.needleFiles }, + } + } + if (task.id === 'ptc-sdk-load-order') { + return { ...base, + sdk: ['ops_inspect_local_file'], + answer: { path: 'answer.txt', content: exports.matches.join('\n') }, + verify: { api: 'ops_grep', loader: 'ops_load_sdk', needed: ['grep', 'write_file'] }, + } + } + return { ...base, + sdk: ['ops_inspect_local_file', 'ops_read_files', 'ops_glob', 'ops_edit_file', 'ops_write_file'], + answer: { path: 'answer.txt', content: String(exports.source) }, + verify: { winner: String(exports.source), editApi: 'ops_edit_file' }, + } +} + +// ---------------------------------------------------------------- envelope helpers + +/** One program: a note plus the calls it dispatched, in order. */ +const program = (note, ...calls) => ({ note, calls }) +/** One dispatched call and the envelope that came back. */ +const call = (api, args, response) => ({ api, args, response }) +/** A completed envelope. */ +const ok = (data) => ({ status: 'completed', data: data ?? {} }) +/** A partial page whose continuation cue is its nextOffset. */ +const partialPage = (nextOffset, total) => ({ status: 'partial', data: { lineCount: 200, nextOffset, total } }) +/** The final page: nothing follows it. */ +const completePage = (total) => ({ status: 'completed', data: { lineCount: 200, total } }) +/** A final refusal. */ +const refused = (code, message) => ({ status: 'rejected', error: { code, message, effectState: 'none', retry: 'never' } }) +/** The lock is busy; the same call may land once the state changes. */ +const lockBusy = () => ({ status: 'error', error: { code: 'JOB_BUSY', effectState: 'none', retry: 'bounded-transient', message: 'the update lock is held by a concurrent writer; re-issue the same edit when it releases' } }) +/** The input is wrong; re-issuing it unchanged gains nothing. */ +const invalid = (message) => ({ status: 'error', error: { code: 'INVALID_ARGUMENT', effectState: 'none', retry: 'after-input-change', message } }) + +// ---------------------------------------------------------------- the scripted tracks + +/** `ptc-paged-lookup`: cursor continuation, verbatim re-read, early stop. */ +function pagingTracks(entry) { + const path = entry.verify.path + const total = entry.exports.totalLines + const answer = entry.answer.content + const read = (offset, response) => call('ops_inspect_local_file', { path, offset, limit: 200 }, response) + const loadWriter = program('load the writer for the answer', call('ops_load_sdk', { tools: ['write_file'] }, ok({ loaded: ['write_file'] }))) + const write = (content) => program('write the answer', call('ops_write_file', { path: 'answer.txt', content }, ok())) + return [ + { id: 'ptc-paged-cursor-pass', taskId: entry.id, declared: 'pass', answer, programs: [ + program('read page 1; the envelope carries a nextOffset', read(1, partialPage(201, total))), + program('the next program continues at the cursor on the same path', read(201, partialPage(401, total))), + program('the last page is complete, so the read chain stops', read(401, completePage(total))), + loadWriter, + write(answer), + ] }, + { id: 'ptc-paged-cursor-retry', taskId: entry.id, declared: 'fail', answer: '1 nothing found', programs: [ + program('read page 1; the envelope carries a nextOffset', read(1, partialPage(201, total))), + program('re-read page 1 instead of continuing at the cursor', read(1, partialPage(201, total))), + loadWriter, + write('1 nothing found'), + ] }, + { id: 'ptc-paged-cursor-early', taskId: entry.id, declared: 'fail', answer: 'no match in the log', programs: [ + program('read page 1; the envelope carries a nextOffset', read(1, partialPage(201, total))), + loadWriter, + write('no match in the log'), + ] }, + ] +} + +/** `ptc-catalog-recover`: an invented name is refused; the recovery either uses an advertised API or invents another one. */ +function catalogTracks(entry) { + const needle = entry.verify.needle + const unknown = (name) => refused('UNKNOWN_API', name + ' is not in this SDK. Available APIs: ' + entry.sdk.join(', ') + '.') + const search = (api) => call(api, { pattern: needle, path: 'src' }, null) + return [ + { id: 'ptc-catalog-recover-pass', taskId: entry.id, declared: 'pass', answer: entry.answer.content, programs: [ + program('the remembered name is refused, so the same program uses an advertised one', + { ...search('ops_find'), response: unknown('ops_find') }, + call('ops_grep', { pattern: needle, path: 'src' }, ok({ matches: entry.exports.matches }))), + program('write the answer', call('ops_write_file', { path: 'answer.txt', content: entry.answer.content }, ok())), + ] }, + { id: 'ptc-catalog-nonexistent-recover', taskId: entry.id, declared: 'fail', answer: 'src/guess.js', programs: [ + program('the remembered name is refused', { ...search('ops_find'), response: unknown('ops_find') }), + program('chase a second invented name instead of an advertised one', { ...search('ops_searchall'), response: unknown('ops_searchall') }), + program('write a guess', call('ops_write_file', { path: 'answer.txt', content: 'src/guess.js' }, ok())), + ] }, + ] +} + +/** `ptc-sdk-load-order`: load the missing tool, or retry the call unchanged. */ +function sdkLoadTracks(entry) { + const needle = entry.task.fixture.params.needle + const notInSdk = () => refused('NOT_IN_SDK', 'ops_grep is not in the current SDK; load it with tools.ops_load_sdk({tools:["grep"]}) — it becomes callable in the next program.') + const grep = (response) => call('ops_grep', { pattern: needle, path: 'src' }, response) + return [ + { id: 'ptc-sdk-load-then-use-pass', taskId: entry.id, declared: 'pass', answer: entry.answer.content, programs: [ + program('grep is refused, so load it in the same program', + grep(notInSdk()), + call('ops_load_sdk', { tools: ['grep', 'write_file'] }, ok({ loaded: ['grep', 'write_file'] }))), + program('the next program may use grep and write the answer', + grep(ok({ matches: entry.exports.matches })), + call('ops_write_file', { path: 'answer.txt', content: entry.answer.content }, ok())), + ] }, + { id: 'ptc-sdk-retry-without-load', taskId: entry.id, declared: 'fail', answer: 'src/nothing.js', programs: [ + program('grep is refused', grep(notInSdk())), + program('retry grep unchanged', grep(notInSdk())), + program('load only the writer', call('ops_load_sdk', { tools: ['write_file'] }, ok({ loaded: ['write_file'] }))), + program('write a guess', call('ops_write_file', { path: 'answer.txt', content: 'src/nothing.js' }, ok())), + ] }, + ] +} + +/** `ptc-wait-dispatch`: batched reads, a lock pause that must be waited out, an immediate counter-example, a no-progress retry. */ +function waitTracks(entry) { + const winner = entry.verify.winner + const find = '"MAX_UPLOAD_MB": ' + entry.exports.value + const replace = '"MAX_UPLOAD_MB": ' + 900 + const revision = 'rev-' + entry.exports.value + const layers = ['README.md', 'config/defaults.json', 'config/production.json', winner] + const readLayers = program('read every layer together', ...layers.map((file) => call('ops_inspect_local_file', { path: file }, ok({ file, revision: file === winner ? revision : 'rev-' + file.length })))) + const edit = (expectedRevision, response) => call('ops_edit_file', { path: winner, find, replace, ...(expectedRevision === undefined ? {} : { expectedRevision }) }, response) + const write = program('write the answer', call('ops_write_file', { path: 'answer.txt', content: winner }, ok())) + return [ + { id: 'ptc-wait-lock-pass', taskId: entry.id, declared: 'pass', answer: winner, programs: [ + readLayers, + program('edit the winner; the lock is held by another writer', edit(revision, lockBusy())), + program('the lock released; the same edit lands', edit(revision, ok({ changed: 1, revision: 'rev-b2' }))), + write, + ] }, + { id: 'ptc-wait-immediate-pass', taskId: entry.id, declared: 'pass', answer: winner, programs: [ + readLayers, + program('edit the winner; it lands at once', edit(revision, ok({ changed: 1, revision: 'rev-b2' }))), + write, + ] }, + { id: 'ptc-wait-no-progress', taskId: entry.id, declared: 'fail', answer: 'config/defaults.json', programs: [ + readLayers, + program('edit without the revision the read returned', edit(undefined, invalid('expectedRevision is required: read the current file state first'))), + program('retry the same broken edit unchanged', edit(undefined, invalid('expectedRevision is required: read the current file state first'))), + program('write a guess', call('ops_write_file', { path: 'answer.txt', content: 'config/defaults.json' }, ok())), + ] }, + ] +} + +/** + * Build the scripted trajectories for the loaded task descriptors. + * @param {object[]} tasks - the descriptors from {@link loadTrackTasks}. + * @returns {object[]} ten tracks: three paging, two catalog, two load, three wait. + */ +export function scriptedTracks(tasks) { + return tasks.flatMap((entry) => { + if (entry.id === 'ptc-paged-lookup') return pagingTracks(entry) + if (entry.id === 'ptc-catalog-recover') return catalogTracks(entry) + if (entry.id === 'ptc-sdk-load-order') return sdkLoadTracks(entry) + return waitTracks(entry) + }) +} diff --git a/bench/lib/tracks.mjs b/bench/lib/tracks.mjs new file mode 100644 index 00000000..88de8af3 --- /dev/null +++ b/bench/lib/tracks.mjs @@ -0,0 +1,299 @@ +/** + * The V4 behavior-track verifier (#993): it judges one scripted trajectory — + * an ordered list of programs, each holding the tool calls it dispatched and the + * envelopes that came back — against the fixed facts of its task, and it says + * which fact failed when the verdict is fail. It never talks to a model and never + * spawns a worker: V5 will feed real captured trajectories through the same + * functions, so the judging rules must not depend on anything but the data. + * + * Two rules are shared by every task and live here once: + * + * - A verbatim re-issue (same API, same arguments) is a no-progress retry when the + * earlier envelope had already delivered a result (completed, or a partial page + * whose data is final for that request) or refused it with an input-side fix + * (retry never / after-input-change) and nothing changed in between. It is a + * real wait when the envelope asked for a state-side pause + * (after-state-change / bounded-transient), and it is a deliberate follow-through + * when a completed ops_load_sdk landed between the two calls. + * - A correction opportunity is a response the model had to act on: a + * rejection, an error, a conflict, or a partial page carrying a nextOffset. + * Recovery counts only tracks that had at least one; a selection with none + * reports an explicit no-denominator line instead of 100%. + * + * The scripted envelopes follow the shipped outcome vocabulary (status / + * error.code / error.retry). Two codes are corpus markers, not shipped codes: + * UNKNOWN_API and NOT_IN_SDK stand for the honest feedback the real SDK gives + * when a name is absent — until V5 replaces them with real captured envelopes. + * + * @module dsh-ops/bench/tracks + */ + +/** Responses that delivered their data: a verbatim re-issue after one is waste. */ +const DELIVERED = ['completed', 'partial'] +/** Retry advice that asks the environment (not the input) to change first. */ +const TRANSIENT_RETRY = ['after-state-change', 'bounded-transient'] +/** Responses a model must act on; each one is a correction opportunity. */ +const SIGNAL_STATUSES = ['rejected', 'error', 'conflict'] + +/** Clamp a fact message so a verdict stays readable in a report. */ +const short = (text) => { + const value = String(text) + return value.length > 80 ? value.slice(0, 77) + '...' : value +} + +/** + * Flatten one trajectory into ordered steps. + * @param {object} track - the scripted trajectory. + * @returns {{index: number, program: number, api: string, args: object, response: object}[]} every call in order. + */ +function stepsOf(track) { + const steps = [] + track.programs.forEach((program, programIndex) => { + for (const call of program.calls ?? []) { + steps.push({ index: steps.length, program: programIndex + 1, api: String(call.api), args: call.args ?? {}, response: call.response ?? {} }) + } + }) + return steps +} + +/** The dedupe key of one step: same API, same arguments. */ +const stepKey = (step) => step.api + ' ' + JSON.stringify(step.args) + +/** + * Detect verbatim re-issues: no-progress retries become facts, real waits are counted. + * @param {{index: number, program: number, api: string, args: object, response: object}[]} steps - every call in order. + * @param {(fact: string) => void} add - record one failed fact. + * @returns {number} how many legitimate waits the trajectory spent. + */ +function scanReissues(steps, add) { + let waits = 0 + const previous = new Map() + for (const step of steps) { + const key = stepKey(step) + const earlier = previous.get(key) + previous.set(key, step) + if (earlier === undefined) continue + const response = earlier.response ?? {} + const between = steps.slice(earlier.index + 1, step.index) + const loadedInBetween = between.some((other) => other.api === 'ops_load_sdk' && other.response?.status === 'completed') + if (loadedInBetween) continue // deliberate follow-through after fixing the SDK state + if (DELIVERED.includes(response.status)) { + add('no-progress retry: ' + step.api + ' ' + short(stepKey(step).slice(step.api.length + 1)) + ' was re-issued verbatim after its result had already been delivered') + } else if (TRANSIENT_RETRY.includes(response.error?.retry)) { + waits += 1 // the envelope asked for a pause, and the model waited instead of re-routing + } else { + add('no-progress retry: ' + step.api + ' was re-issued verbatim although the envelope said retry: ' + String(response.error?.retry) + ' — change the input or load the tool first') + } + } + return waits +} + +/** + * Count the correction opportunities in one trajectory. + * @param {{index: number, program: number, api: string, args: object, response: object}[]} steps - every call in order. + * @returns {number} how many responses the model had to act on. + */ +function countOpportunities(steps) { + return steps.filter((step) => { + const response = step.response ?? {} + if (SIGNAL_STATUSES.includes(response.status)) return true + if (response.status === 'partial' && Number.isInteger(Number(response.data?.nextOffset)) && Number(response.data?.nextOffset) > 0) return true + return false + }).length +} + +/** + * Judge one scripted trajectory against its task's fixed facts. + * @param {object} track - the trajectory: {id, taskId, declared, programs, answer}. + * @param {object} task - the task descriptor from the corpus loader: {id, sdk, budget, answer, verify}. + * @returns {{id: string, taskId: string, declared: string, verdict: string, matched: boolean, mismatches: string[], opportunities: number, waits: number, recovered: boolean|null}} the verdict. + * @throws {Error} when the track and the task do not match, or no verifier is registered. + */ +export function judgeTrack(track, task) { + if (!task || task.id !== track.taskId) { + throw new Error('judgeTrack: track "' + String(track?.id) + '" names task "' + String(track?.taskId) + '" but was handed task "' + String(task?.id) + '"') + } + const verifier = TASK_VERIFIERS[task.id] + if (verifier === undefined) throw new Error('judgeTrack: no verifier is registered for task "' + task.id + '"') + const steps = stepsOf(track) + const mismatches = [] + const add = (fact) => mismatches.push(fact) + verifier(track, task, steps, add) + const waits = scanReissues(steps, add) + if (String(track.answer ?? '').trim() !== String(task.answer.content).trim()) { + add('answer.txt: expected "' + short(task.answer.content) + '" but the track wrote "' + short(track.answer) + '"') + } + if (track.programs.length > task.budget.maxPrograms) { + add('the track spent ' + track.programs.length + ' programs, over the fixed budget of ' + task.budget.maxPrograms) + } + const opportunities = countOpportunities(steps) + const verdict = mismatches.length === 0 ? 'pass' : 'fail' + return { + id: track.id, + taskId: track.taskId, + declared: track.declared, + verdict, + matched: verdict === track.declared, + mismatches, + opportunities, + waits, + recovered: opportunities === 0 ? null : verdict === 'pass', + } +} + +/** + * Summarise recovery over a selection of verdicts. + * @param {object[]} verdicts - judgeTrack results. + * @returns {{tracks: number, withOpportunity: number, recovered: number, rate: number|null, hasDenominator: boolean, note: string}} the accounting. + */ +export function recoverySummary(verdicts) { + const withOpportunity = verdicts.filter((verdict) => verdict.opportunities > 0) + const recovered = withOpportunity.filter((verdict) => verdict.recovered === true).length + const hasDenominator = withOpportunity.length > 0 + return { + tracks: verdicts.length, + withOpportunity: withOpportunity.length, + recovered, + rate: hasDenominator ? recovered / withOpportunity.length : null, + hasDenominator, + note: hasDenominator + ? 'recovered ' + recovered + ' of ' + withOpportunity.length + ' tracks that had a correction opportunity' + : 'no denominator: no track in this selection had a correction opportunity, so no recovery rate is reported (never 100%)', + } +} + +// ---------------------------------------------------------------- the four fact verifiers + +/** + * `ptc-paged-lookup`: a partial page must be continued at its nextOffset in the next + * program on the same path, and a complete page ends the read chain. + */ +function verifyPaged(track, task, steps, add) { + const spec = task.verify + const reads = steps.filter((step) => step.api === 'ops_inspect_local_file' && step.args.path === spec.path) + if (reads.length === 0) { + add('no read of ' + spec.path + ' ever happened') + return + } + let cursor = null + let cursorProgram = 0 + let completeAt = null + for (const read of reads) { + const offset = Number(read.args.offset ?? 1) + if (completeAt !== null) { + add('read of ' + spec.path + ' at offset ' + offset + ' after the complete page — a complete page ends the read chain') + continue + } + if (cursor !== null && offset !== cursor) { + add('read at offset ' + offset + ' ignored the pending cursor (the envelope said nextOffset ' + cursor + ')') + } + if (cursor !== null && offset === cursor && read.program <= cursorProgram) { + add('the continuation at offset ' + offset + ' ran in program ' + read.program + ', not in a program after the one that set the cursor') + } + if (read.response.status === 'partial') { + const next = Number(read.response.data?.nextOffset) + if (!Number.isInteger(next) || next <= offset) { + add('partial page at offset ' + offset + ' carried no advancing nextOffset') + } else { + cursor = next + cursorProgram = read.program + } + } else if (read.response.status === 'completed') { + completeAt = read.program + cursor = null + } else { + add('unexpected status "' + String(read.response.status) + '" while reading ' + spec.path) + } + } + if (cursor !== null) { + add('the read chain stopped with a pending cursor at offset ' + cursor + ' — the next program must continue at that offset') + } + if (completeAt === null) { + add('no complete page ever ended the read chain of ' + spec.path) + } +} + +/** + * `ptc-catalog-recover`: after the wrong call was rejected, every following call + * must use an advertised API, and one advertised alternative must have worked. + */ +function verifyCatalog(track, task, steps, add) { + const rejectedAt = steps.findIndex((step) => step.response.status === 'rejected' && step.response.error?.code === 'UNKNOWN_API') + if (rejectedAt < 0) { + add('the catalog-error track never made the wrong call (no UNKNOWN_API rejection happened)') + return + } + const after = steps.slice(rejectedAt + 1) + for (const step of after) { + if (!task.sdk.includes(step.api)) { + add('call "' + step.api + '" after the wrong input is not in the advertised SDK set (known: ' + task.sdk.join(', ') + ')') + } + } + if (!after.some((step) => task.sdk.includes(step.api) && step.response.status === 'completed')) { + add('no advertised alternative was used after the wrong input') + } +} + +/** + * `ptc-sdk-load-order`: the needed tool must be loaded before it is used, and a + * completed use may only run in a program after the one that loaded it. + */ +function verifySdkLoad(track, task, steps, add) { + const spec = task.verify + const uses = steps.filter((step) => step.api === spec.api) + if (!uses.some((step) => step.response.status === 'rejected' && step.response.error?.code === 'NOT_IN_SDK')) { + add('the track never hit the not-loaded rejection of ' + spec.api) + } + const load = steps.find((step) => step.api === spec.loader && step.response.status === 'completed') + const loadsTools = load === undefined ? [] : (Array.isArray(load.args?.tools) ? load.args.tools : []) + if (load === undefined || !spec.needed.every((name) => loadsTools.includes(name))) { + add(spec.loader + ' was never called with the needed names (' + spec.needed.join(', ') + ')') + } + const firstUse = uses.find((step) => step.response.status === 'completed') + if (firstUse === undefined) { + add(spec.api + ' was never used successfully, so the answer cannot come from it') + return + } + if (load === undefined || firstUse.index < load.index) { + add('a completed ' + spec.api + ' call ran before ' + spec.loader + ' loaded it') + } else if (firstUse.program <= load.program) { + add('a completed ' + spec.api + ' call ran in program ' + firstUse.program + ', not in a program after the load (program ' + load.program + ')') + } +} + +/** + * `ptc-wait-dispatch`: the independent layer reads are batched in one program, the + * edit must await the read result through expectedRevision, and a bounded-transient + * pause is a real wait, never a flagged retry. + */ +function verifyWaitDispatch(track, task, steps, add) { + const spec = task.verify + const edits = steps.filter((step) => step.api === spec.editApi && step.args.path === spec.winner) + if (edits.length === 0) { + add('the winning layer ' + spec.winner + ' was never edited') + return + } + const revision = edits[0].args?.expectedRevision + const read = steps.find((step) => step.api === 'ops_inspect_local_file' && step.args.path === spec.winner + && step.response.status === 'completed' && step.response.data?.revision === revision) + if (typeof revision !== 'string' || read === undefined || read.index >= edits[0].index) { + add('the edit did not await the read result: expectedRevision ' + JSON.stringify(revision ?? null) + ' does not come from an earlier completed read of ' + spec.winner) + } + const perProgram = new Map() + for (const step of steps) { + if (step.api === 'ops_inspect_local_file' && step.response.status === 'completed') { + perProgram.set(step.program, (perProgram.get(step.program) ?? 0) + 1) + } + } + if (![...perProgram.values()].some((count) => count >= 2)) { + add('the independent layer reads were not dispatched together in one program') + } +} + +/** The verifier registry: one entry per V4 behavior task (#993). */ +export const TASK_VERIFIERS = { + 'ptc-paged-lookup': verifyPaged, + 'ptc-catalog-recover': verifyCatalog, + 'ptc-sdk-load-order': verifySdkLoad, + 'ptc-wait-dispatch': verifyWaitDispatch, +} diff --git a/bench/tasks/ptc-catalog-recover.task.json b/bench/tasks/ptc-catalog-recover.task.json new file mode 100644 index 00000000..1664a739 --- /dev/null +++ b/bench/tasks/ptc-catalog-recover.task.json @@ -0,0 +1,12 @@ +{ + "id": "ptc-catalog-recover", + "title": "Locate the five files that mention a token", + "prompt": "Exactly five files under src/ contain the token NEEDLE_CATALOG and no other file does. Write their repo-relative paths to answer.txt, one per line, in sorted order, nothing else.", + "fixture": { + "generator": "grepTree", + "params": { "seed": 5, "dirs": 6, "filesPerDir": 12, "linesPerFile": 30, "needle": "NEEDLE_CATALOG", "needleFiles": 5 } + }, + "verify": [{ "type": "fileLines", "path": "answer.txt", "expect": "matches" }], + "repeat": 1, + "timeoutMs": 900000 +} diff --git a/bench/tasks/ptc-paged-lookup.task.json b/bench/tasks/ptc-paged-lookup.task.json new file mode 100644 index 00000000..96260c0a --- /dev/null +++ b/bench/tasks/ptc-paged-lookup.task.json @@ -0,0 +1,12 @@ +{ + "id": "ptc-paged-lookup", + "title": "Find the marked receipt in a 600-line log", + "prompt": "logs/stream.log is a dispatch receipt log with one record per line, formatted LEVEL|STAMP|PAYLOAD. Exactly one line's payload contains the token TARGET-. Find that line and write to answer.txt its 1-based line number, then a single space, then that line's payload text. Nothing else in the file.", + "fixture": { + "generator": "pagedLog", + "params": { "seed": 41, "lines": 600, "markerAt": 350 } + }, + "verify": [{ "type": "fileEquals", "path": "answer.txt", "expect": "answer" }], + "repeat": 1, + "timeoutMs": 900000 +} diff --git a/bench/tasks/ptc-sdk-load-order.task.json b/bench/tasks/ptc-sdk-load-order.task.json new file mode 100644 index 00000000..46ffd9f9 --- /dev/null +++ b/bench/tasks/ptc-sdk-load-order.task.json @@ -0,0 +1,12 @@ +{ + "id": "ptc-sdk-load-order", + "title": "Locate the three files that mention a token", + "prompt": "Exactly three files under src/ contain the token NEEDLE_SDKLOAD and no other file does. Write their repo-relative paths to answer.txt, one per line, in sorted order, nothing else.", + "fixture": { + "generator": "grepTree", + "params": { "seed": 9, "dirs": 5, "filesPerDir": 10, "linesPerFile": 24, "needle": "NEEDLE_SDKLOAD", "needleFiles": 3 } + }, + "verify": [{ "type": "fileLines", "path": "answer.txt", "expect": "matches" }], + "repeat": 1, + "timeoutMs": 900000 +} diff --git a/bench/tasks/ptc-wait-dispatch.task.json b/bench/tasks/ptc-wait-dispatch.task.json new file mode 100644 index 00000000..e21f0792 --- /dev/null +++ b/bench/tasks/ptc-wait-dispatch.task.json @@ -0,0 +1,15 @@ +{ + "id": "ptc-wait-dispatch", + "title": "Move the effective setting to a new value in the one file that owns it", + "prompt": "The application reads MAX_UPLOAD_MB from the three JSON layers documented in README.md; a later layer replaces the same key from an earlier one, and packages/*/config.json is not merged. Change the effective value to 900 by editing only the one file that currently provides it, then write that file's repo-relative path to answer.txt, nothing else.", + "fixture": { + "generator": "configTree", + "params": { "seed": 21, "packages": 8, "env": "production" } + }, + "verify": [ + { "type": "fileEquals", "path": "answer.txt", "expect": "source" }, + { "type": "fileContains", "path": "config/production.local.json", "text": "\"MAX_UPLOAD_MB\": 900" } + ], + "repeat": 1, + "timeoutMs": 900000 +} diff --git a/bench/tracks.mjs b/bench/tracks.mjs new file mode 100644 index 00000000..10e93b1c --- /dev/null +++ b/bench/tracks.mjs @@ -0,0 +1,106 @@ +/** + * The `tracks` subcommand (#993): judge the scripted V4 behavior trajectories — + * paged cursor, catalog recovery, load order, wait/multi-dispatch — against their + * fixed task facts, and report the recovery accounting with its denominator rule. + * + * This is verifier self-proof before V5: every scripted track declares what the + * verifier should say, the command exits non-zero the moment a verdict disagrees, + * and the recovery line says "no denominator" instead of 100% when a selection + * contains no track that ever had a correction opportunity. + * + * @module dsh-ops/bench/tracks-cli + */ + +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { loadTrackTasks, scriptedTracks, TRACK_TASK_IDS } from './lib/track-corpus.mjs' +import { judgeTrack, recoverySummary } from './lib/tracks.mjs' + +/** The flags that take one value. */ +const FLAGS_WITH_VALUE = ['--task', '--track'] + +/** The command's own help, printed for `tracks --help`. */ +export function tracksUsage() { + return [ + 'usage: node bench/cli.mjs tracks [--task ] [--track ]', + '', + 'Judges the scripted V4 behavior trajectories (#993) against their task facts.', + 'No model request and no worker spawn: this is the verifier self-proof step.', + '', + '--task only the tracks of one task (' + TRACK_TASK_IDS.join(', ') + ')', + '--track one scripted trajectory by id', + '', + 'Exit codes: 0 every verdict matched its declared expectation · 1 a verdict disagreed · 2 usage error', + ].join('\n') +} + +/** + * Run the command. + * @param {string[]} argv - the arguments after `tracks`. + * @param {{out?: (line: string) => void, err?: (line: string) => void}} [io] - the output sinks. + * @returns {Promise} the exit code: 0 agreed, 1 disagreed, 2 usage error. + */ +export async function main(argv = [], io = {}) { + const out = io.out ?? ((line) => console.log(line)) + const err = io.err ?? ((line) => console.error(line)) + let taskId + let trackId + for (let index = 0; index < argv.length; index += 1) { + const flag = argv[index] + if (flag === '-h' || flag === '--help') { + out(tracksUsage()) + return 0 + } + if (FLAGS_WITH_VALUE.includes(flag)) { + const value = argv[index + 1] + if (value === undefined || value.startsWith('--')) { + err('bench tracks: ' + flag + ' needs a value') + err('run "node bench/cli.mjs tracks --help" for usage') + return 2 + } + if (flag === '--task') taskId = value + else trackId = value + index += 1 + continue + } + err('bench tracks: unknown option "' + flag + '"') + err('run "node bench/cli.mjs tracks --help" for usage') + return 2 + } + const tasks = loadTrackTasks() + if (taskId !== undefined && !TRACK_TASK_IDS.includes(taskId)) { + err('bench tracks: unknown task "' + taskId + '" (known: ' + TRACK_TASK_IDS.join(', ') + ')') + return 2 + } + let tracks = scriptedTracks(tasks) + if (taskId !== undefined) tracks = tracks.filter((track) => track.taskId === taskId) + if (trackId !== undefined) { + if (!tracks.some((track) => track.id === trackId)) { + err('bench tracks: unknown track "' + trackId + '" (known: ' + tracks.map((track) => track.id).join(', ') + ')') + return 2 + } + tracks = tracks.filter((track) => track.id === trackId) + } + const verdicts = tracks.map((track) => judgeTrack(track, tasks.find((task) => task.id === track.taskId))) + for (const verdict of verdicts) { + out(verdict.id + ' (' + verdict.taskId + '): declared ' + verdict.declared + ', judged ' + verdict.verdict + + ', opportunities ' + verdict.opportunities + ', waits ' + verdict.waits) + for (const fact of verdict.mismatches) out(' - ' + fact) + } + const summary = recoverySummary(verdicts) + if (summary.hasDenominator) { + out('recovery: ' + summary.recovered + '/' + summary.withOpportunity + ' tracks with a correction opportunity recovered (' + Math.round(summary.rate * 100) + '%)') + } else { + out('recovery: no denominator (0 tracks had a correction opportunity, so no rate is reported)') + } + out(summary.note) + const disagreed = verdicts.filter((verdict) => !verdict.matched).length + out('tracks: ' + verdicts.length + ' judged, ' + (verdicts.length - disagreed) + ' matched the declared expectation') + return disagreed === 0 ? 0 : 1 +} + +// Only when executed, not when imported by the suite or the dispatcher. +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url))) { + process.exitCode = await main(process.argv.slice(2)) +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 83f6202a..f7bb7795 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -18,6 +18,9 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm - New OPS-PTC acceptance test infrastructure: `test/lib/ptc-entry.mjs` boots one preset group on the real agent loop with the shipped `ops-ptc` row, inside each suite's throwaway `DSH_HOME`, and records the environment a measurement must name — plugin commit, host version, platform, the worker's sha256 with its staged receipt, sandbox mode, the row's resolved switches and the actual `agentPreset`. The minimal case is one scripted provider request proving the top-level transport is `run_code`; a group that lands on Standard, on a native executor, or without an executable worker is reported invalid with its own reason and carries no comparable output, so a wrong configuration can no longer pass as OPS-PTC evidence. `test/progressive-loop.test.mjs` asserts all of this (#990; see `docs/isolated-ptc-entry.md`). - 新增 OPS-PTC 验收测试基础设施:`test/lib/ptc-entry.mjs` 在每套件独占的临时 `DSH_HOME` 内,用正式 `ops-ptc` 行装配真实 agent loop 的一个组,并记录测量必须交代的环境——插件 commit、宿主版本、平台、worker 的 sha256 及 staged 回执、沙箱模式、行内实际开关与实际 `agentPreset`。最小用例是一次脚本化 provider 请求,先证明顶层传输是 `run_code`;落在 Standard、native executor 或无可执行 worker 上的组按各自原因判「无效」且不带任何可比结果,错误配置不再能冒充 OPS-PTC 证据。`test/progressive-loop.test.mjs` 对此逐项断言(#990,见 `docs/隔离测试入口.md`)。 +- New bench-only behavior corpus for the OPS-PTC cards (#993): four tasks in `bench/tasks/` — paged cursor continuation, catalog-error recovery, load-before-use and wait/multi-dispatch — whose prompts state only the business goal, plus a `pagedLog` fixture family whose answer derives from its bytes. `bench/lib/tracks.mjs` judges a scripted trajectory against each task's fixed facts (cursor advance in the next program on the same path, complete-page stop, advertised-set recovery, load order, revision-awaited edits, verbatim no-progress retries versus legitimate bounded-transient waits) and names the fact that did not hold. `node bench/cli.mjs tracks` runs the ten scripted pass/fail trajectories and exits non-zero when a verdict disagrees; the recovery rate counts only trajectories that actually had a correction opportunity and prints an explicit no-denominator line instead of 100%. `test/bench-tracks.test.mjs` pins both states. `bench/` ships in no npm package; no plugin behaviour changed. +- 新增仅限仓库内的 OPS-PTC 行为语料(#993):`bench/tasks/` 下四个任务——分页游标续读、目录错误纠偏、先加载后调用、等待/多派发——题面只写业务目标,另加一个答案可从字节反推的 `pagedLog` 夹具族。`bench/lib/tracks.mjs` 按各任务固定事实判定脚本化轨迹(下一程序同路径续读光标、末页即停、纠偏只认当时可用集合、加载先于调用、编辑等待读取返回的 revision、原样重读属无进展重试而 bounded-transient 等待不算),失败输出指明是哪项事实不符。`node bench/cli.mjs tracks` 跑十条应过/应败轨迹,任何判定与声明不符即退非零;恢复率只统计真的出现过纠偏机会的轨迹,无分母时明说「no denominator」而不是报 100%。`test/bench-tracks.test.mjs` 把两种状态都钉住。`bench/` 不随 npm 包发布,插件行为未变。 + ## [0.5.5] — 2026-11-13 — Patch release - A loader-less PTC scope no longer registers the configured `allowedTools` pool: with dynamic loading off nothing could ever select those APIs, and a host assembly without a step tool set declared all sixteen in `tools:sdk`. Resident publication, the default inspect + loader set and selection behavior are unchanged (#887). diff --git a/test/bench-fixtures.test.mjs b/test/bench-fixtures.test.mjs index 24266df0..f5a51d59 100644 --- a/test/bench-fixtures.test.mjs +++ b/test/bench-fixtures.test.mjs @@ -21,8 +21,8 @@ import { CPU_PROCESS_TIMEOUT_MS, assert, report, test, withTempDir } from './lib import { brokenProject, configTree, deepCorpus, deepModule, deriveConfigTree, deriveDeepModule, deriveGrepTree, deriveLargeCsv, deriveLogStats, deriveNoisySuite, deriveRenameTree, - deriveSymbolTree, gatePipeline, generate, GENERATORS, grepTree, largeCsv, logStats, - mulberry32, noisySuite, renameTree, symbolTree, + derivePagedLog, deriveSymbolTree, gatePipeline, generate, GENERATORS, grepTree, largeCsv, logStats, + mulberry32, noisySuite, pagedLog, renameTree, symbolTree, } from '../bench/lib/fixtures/index.mjs' const LABEL = 'bench-fixtures' @@ -40,12 +40,13 @@ const SMALL = { noisySuite: { seed: 29, tests: 200 }, symbolTree: { seed: 31, files: 6 }, gatePipeline: { seed: 53, gates: 4, probes: 3, linesPerGate: 4 }, + pagedLog: { seed: 41, lines: 120, markerAt: 60 }, } /** The complete registry: every family builder, in the order `GENERATORS` lists them. */ const ALL_GENERATORS = [ 'brokenProject', 'configTree', 'deepCorpus', 'deepModule', 'gatePipeline', 'grepTree', - 'largeCsv', 'logStats', 'noisySuite', 'renameTree', 'symbolTree', + 'largeCsv', 'logStats', 'noisySuite', 'pagedLog', 'renameTree', 'symbolTree', ] /** Write one generated file into a workspace, creating its parents. */ @@ -128,6 +129,16 @@ await test('logStats exports the ERROR counts recomputed from its own bytes', () assert.deepEqual(exports.report, [...exports.report].sort(), 'the report must be in file-name order') }) +await test('pagedLog exports the marked line recomputed from its own bytes', () => { + const { files, exports } = pagedLog(SMALL.pagedLog) + assert.equal(files.length, 1) + assert.equal(files[0].path, 'logs/stream.log') + assert.deepEqual(derivePagedLog(files), exports, 'the exported answer must be of the real bytes') + assert.ok(files[0].content.split('TARGET-').length === 2, 'the marker must appear exactly once') + assert.throws(() => pagedLog({ ...SMALL.pagedLog, markerAt: 0 }), RangeError) + assert.throws(() => pagedLog({ ...SMALL.pagedLog, markerAt: SMALL.pagedLog.lines + 1 }), RangeError) +}) + await test('configTree separates the winning layer from every look-alike', () => { const { files, exports } = configTree(SMALL.configTree) const read = (name) => files.find((file) => file.path === name)?.content ?? '' @@ -283,7 +294,7 @@ await test('generate() rejects an unknown generator by name', () => { // exists but was never registered (or never re-exported) is unreachable, and a // duplicate or a stray extra key has to fail here rather than in a task run. assert.deepEqual(Object.keys(GENERATORS).sort(), ALL_GENERATORS) - assert.equal(Object.keys(GENERATORS).length, 11) + assert.equal(Object.keys(GENERATORS).length, 12) assert.deepEqual(Object.keys(SMALL).sort(), ALL_GENERATORS, 'the small-parameter table must cover every registered generator') for (const [name, builder] of Object.entries(GENERATORS)) assert.equal(typeof builder, 'function', `${name} must be exported as a builder`) }) diff --git a/test/bench-tracks.test.mjs b/test/bench-tracks.test.mjs new file mode 100644 index 00000000..19e482e1 --- /dev/null +++ b/test/bench-tracks.test.mjs @@ -0,0 +1,206 @@ +/** + * The V4 behavior-track gate (#993): four Flash behavior samples — paged cursor + * continuation, catalog-error recovery, load-before-use, and wait/multi-dispatch — + * each with a business-goal prompt, a scripted pass trajectory and scripted fail + * trajectories, plus the verifier that must tell them apart. + * + * The red step of this suite came first: every module it imports was absent, so the + * first run failed at the import. The green step is the corpus + verifier landing. + * The suite never spawns a worker and never sends a model request: it judges + * scripted trajectories only, which is exactly what the card asks (V5 owns real runs). + * + * @module dsh-ops/test/bench-tracks + */ + +import fs from 'node:fs' +import path from 'node:path' + +import { assert, report, test, withTempDir } from './lib/harness.mjs' +import { TASKS_DIR, loadTasks } from '../bench/lib/tasks.mjs' +import { GENERATORS, generate } from '../bench/lib/fixtures/index.mjs' +import { derivePagedLog } from '../bench/lib/fixtures/paged.mjs' +import { verifyWorkspace } from '../bench/lib/verify.mjs' +import { judgeTrack, recoverySummary } from '../bench/lib/tracks.mjs' +import { loadTrackTasks, scriptedTracks, TRACK_TASK_IDS } from '../bench/lib/track-corpus.mjs' +import { main as cliMain, usage } from '../bench/cli.mjs' + +const LABEL = 'bench-tracks' + +/** Write one generated fixture file into a workspace, creating its parents. */ +const write = (dir, file) => { + const target = path.join(dir, ...file.path.split('/')) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, file.content, 'utf8') +} + +// ------------------------------------------------------------ the corpus + +await test('the four V4 task files validate against the real registries', async () => { + const corpus = loadTasks(TASKS_DIR, { generators: GENERATORS }) + const ids = corpus.map((task) => task.id) + for (const id of TRACK_TASK_IDS) { + const task = corpus.find((entry) => entry.id === id) + assert.ok(task, 'task ' + id + ' must exist in bench/tasks') + assert.ok(!task.prompt.toLowerCase().includes('verifier'), 'the ' + id + ' prompt must not name the verifier') + assert.ok(!/nextoffset|cursor|load_sdk/i.test(task.prompt), 'the ' + id + ' prompt must not teach the tool steps') + } + assert.equal(ids.length, new Set(ids).size, 'task ids must be unique') +}) + +await test('pagedLog puts the target on the second page and its answer derives from the bytes', () => { + const { files, exports } = generate('pagedLog', { seed: 41, lines: 600, markerAt: 350, pageSize: 200 }) + assert.equal(files.length, 1) + assert.equal(files[0].path, 'logs/stream.log') + assert.deepEqual(derivePagedLog(files), exports, 'the exported answer must be recomputable from the bytes') + assert.ok(exports.markerLine > 200, 'the marker must sit beyond page 1 at the default page size') + assert.ok(exports.markerLine <= 400, 'the marker must sit on page 2 at the default page size') + const lines = files[0].content.split('\n') + assert.ok(lines[exports.markerLine - 1].includes(exports.markerText), 'the marker line must carry the marker payload') + assert.ok(exports.markerText.includes('TARGET-'), 'the marker payload must carry the TARGET token') + assert.equal(exports.answer, exports.markerLine + ' ' + exports.markerText) +}) + +await test('each task workspace verify accepts the correct answer and rejects a wrong one', async () => { + const tasks = await loadTrackTasks() + assert.equal(tasks.length, 4) + for (const entry of tasks) { + await withTempDir('bench-tracks-verify', async (dir) => { + for (const file of entry.files) write(dir, file) + // The wait task also requires the business edit the prompt asks for. + if (entry.id === 'ptc-wait-dispatch') { + const target = path.join(dir, entry.answer.content) + const before = fs.readFileSync(target, 'utf8') + const edited = before.replace('"MAX_UPLOAD_MB": ' + entry.exports.value, '"MAX_UPLOAD_MB": ' + 900) + assert.notEqual(edited, before, 'the edit pattern must match the winner file') + fs.writeFileSync(target, edited) + } + fs.writeFileSync(path.join(dir, 'answer.txt'), entry.answer.content) + const good = await verifyWorkspace(dir, entry.task.verify, entry.exports) + assert.ok(good.passed, entry.id + ': the correct answer must pass every workspace check (got: ' + good.checks.map((c) => c.detail).join(' | ') + ')') + fs.writeFileSync(path.join(dir, 'answer.txt'), 'definitely not the answer') + const bad = await verifyWorkspace(dir, entry.task.verify, entry.exports) + assert.ok(!bad.passed, entry.id + ': a wrong answer must fail the workspace check') + }) + } +}) + +// ------------------------------------------------------------ the verdicts + +const judge = async () => { + const tasks = await loadTrackTasks() + return scriptedTracks(tasks).map((track) => judgeTrack(track, tasks.find((task) => task.id === track.taskId))) +} + +await test('every task has a should-pass and a should-fail trajectory, and the verifier agrees with both', async () => { + const verdicts = await judge() + assert.equal(verdicts.length, 10, 'the scripted corpus holds ten tracks') + for (const id of TRACK_TASK_IDS) { + const own = verdicts.filter((verdict) => verdict.taskId === id) + assert.ok(own.some((v) => v.declared === 'pass'), id + ' needs a should-pass trajectory') + assert.ok(own.some((v) => v.declared === 'fail'), id + ' needs a should-fail trajectory') + } + for (const verdict of verdicts) { + assert.equal(verdict.matched, true, + verdict.id + ': declared ' + verdict.declared + ' but judged ' + verdict.verdict + ' (' + verdict.mismatches.join(' | ') + ')') + } +}) + +await test('the paging verifier names the ignored cursor, the unreached complete page and the wrong answer', async () => { + const verdicts = await judge() + const pass = verdicts.find((v) => v.id === 'ptc-paged-cursor-pass') + assert.equal(pass.verdict, 'pass') + assert.deepEqual(pass.mismatches, []) + assert.ok(pass.opportunities >= 2, 'every partial page is a continuation cue the model must act on') + assert.equal(pass.recovered, true) + const retry = verdicts.find((v) => v.id === 'ptc-paged-cursor-retry') + assert.equal(retry.verdict, 'fail') + assert.match(retry.mismatches.join(' | '), /cursor|offset/i) + assert.match(retry.mismatches.join(' | '), /complete/i) + assert.match(retry.mismatches.join(' | '), /answer/) + const early = verdicts.find((v) => v.id === 'ptc-paged-cursor-early') + assert.equal(early.verdict, 'fail') + assert.match(early.mismatches.join(' | '), /answer/) +}) + +await test('the catalog verifier accepts only advertised alternatives and names the invented one', async () => { + const verdicts = await judge() + const pass = verdicts.find((v) => v.id === 'ptc-catalog-recover-pass') + assert.equal(pass.verdict, 'pass') + assert.equal(pass.opportunities, 1) + assert.equal(pass.recovered, true) + const fail = verdicts.find((v) => v.id === 'ptc-catalog-nonexistent-recover') + assert.equal(fail.verdict, 'fail') + assert.match(fail.mismatches.join(' | '), /ops_searchall/) + assert.match(fail.mismatches.join(' | '), /advertised/) + assert.match(fail.mismatches.join(' | '), /answer/) +}) + +await test('the load verifier demands load_sdk before the next program can use the tool', async () => { + const verdicts = await judge() + const pass = verdicts.find((v) => v.id === 'ptc-sdk-load-then-use-pass') + assert.equal(pass.verdict, 'pass') + assert.equal(pass.opportunities, 1) + assert.equal(pass.recovered, true) + const fail = verdicts.find((v) => v.id === 'ptc-sdk-retry-without-load') + assert.equal(fail.verdict, 'fail') + assert.match(fail.mismatches.join(' | '), /load/) + assert.match(fail.mismatches.join(' | '), /answer/) +}) + +await test('waits are not misjudged: the lock wait passes, the immediate counter-example needs no retry at all', async () => { + const verdicts = await judge() + const wait = verdicts.find((v) => v.id === 'ptc-wait-lock-pass') + assert.equal(wait.verdict, 'pass', 'the lock wait must not be flagged (' + wait.mismatches.join(' | ') + ')') + assert.ok(wait.waits >= 1, 'the bounded-transient re-issue must count as a real wait') + assert.equal(wait.recovered, true) + const immediate = verdicts.find((v) => v.id === 'ptc-wait-immediate-pass') + assert.equal(immediate.verdict, 'pass') + assert.deepEqual(immediate.mismatches, []) + assert.equal(immediate.opportunities, 0) + assert.equal(immediate.recovered, null, 'a track without a correction opportunity has no recovery value') + assert.equal(immediate.waits, 0) + const fail = verdicts.find((v) => v.id === 'ptc-wait-no-progress') + assert.equal(fail.verdict, 'fail') + assert.match(fail.mismatches.join(' | '), /no-progress|verbatim/i) + assert.match(fail.mismatches.join(' | '), /revision/i) +}) + +// ------------------------------------------------------------ recovery accounting + +await test('the recovery rate counts only tracks with a correction opportunity and never invents a denominator', async () => { + const verdicts = await judge() + const summary = recoverySummary(verdicts) + assert.equal(summary.tracks, 10) + assert.equal(summary.withOpportunity, 9, 'only the immediate counter-example has no correction opportunity') + assert.equal(summary.recovered, 4) + assert.equal(summary.rate, 4 / 9) + const onlyImmediate = recoverySummary(verdicts.filter((v) => v.id === 'ptc-wait-immediate-pass')) + assert.equal(onlyImmediate.withOpportunity, 0) + assert.equal(onlyImmediate.rate, null) + assert.equal(onlyImmediate.hasDenominator, false) + assert.ok(onlyImmediate.note.includes('no denominator'), 'a missing denominator must be said, not reported as 100%') +}) + +// ------------------------------------------------------------ the CLI + +await test('bench/cli.mjs lists tracks and the command judges the corpus end to end', async () => { + assert.match(usage(), /\n tracks\s/) + const lines = [] + const io = { out: (line) => lines.push(line), err: (line) => lines.push('ERR ' + line) } + const code = await cliMain(['tracks'], io) + assert.equal(code, 0, lines.join('\n')) + const text = lines.join('\n') + assert.ok(text.includes('ptc-paged-cursor-pass'), 'every scripted track must be reported') + assert.match(text, /recovery: 4\/9|recovery: 44%/) + const scoped = [] + await cliMain(['tracks', '--task', 'ptc-wait-dispatch'], { out: (line) => scoped.push(line), err: (line) => scoped.push('ERR ' + line) }) + assert.ok(scoped.join('\n').includes('ptc-wait-immediate-pass'), '--task must scope the corpus') + const single = [] + await cliMain(['tracks', '--track', 'ptc-wait-immediate-pass'], { out: (line) => single.push(line), err: (line) => single.push('ERR ' + line) }) + assert.ok(single.join('\n').includes('no denominator'), 'the no-retry counter-example must report no denominator') + const bogus = [] + assert.equal(await cliMain(['tracks', '--track', 'nope'], { out: (line) => bogus.push(line), err: (line) => bogus.push('ERR ' + line) }), 2) + assert.match(bogus.join('\n'), /unknown track/) +}) + +report(LABEL) From d0fcd6f1bf6896532c29c18e16464d32f6138d6f Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:02:20 +0800 Subject: [PATCH 7/9] feat(publication): publish the explicit full PTC disclosure tier Consume the D2-parsed disclosurePolicy in the builtin plan slice and the PTC publication: an explicit 'full' row publishes its whole permitted allowedTools pool plus result in the first round with no ops_load_sdk (#987, #979 SS2). Resolve the D1 sign-off item 1 conservatively per #979 SS3 -- and per the unified acceptance rule, which on-demand loading of result forces anyway: the full first round publishes ops_result so refs spilled this step stay consumable. Read-only still removes every mutating API; an empty cap stays at resident + result; residentTools false drops the resident surface but not the tier; duplicated names register once; global.dynamicTools:false keeps its D2 veto end to end and a row that also turns its loader off keeps the tier. On-demand (default) behaviour is letter-identical. PtcConfig.disclosurePolicy stays optional because custom integrations keep the legacy default until #988; the builtin plan carries the parsed value. --- docs/CHANGELOG.md | 2 + docs/configuration-reference.md | 2 +- docs/design-decisions-20261119.md | 54 ++++- ...6\241\345\206\263\347\255\226-20261119.md" | 43 +++- ...15\347\275\256\345\217\202\350\200\203.md" | 2 +- src/presets/builtin.ts | 2 +- src/ptc/config.ts | 4 +- src/publication/ptc.ts | 19 +- test/ptc-declaration-surface.test.mjs | 66 +++++- test/ptc-full-disclosure.test.mjs | 214 ++++++++++++++++++ 10 files changed, 390 insertions(+), 18 deletions(-) create mode 100644 test/ptc-full-disclosure.test.mjs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 3a8cde35..ea2d24f3 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Publish the explicit full PTC disclosure tier (#987): with `disclosurePolicy: 'full'` the `ptc` and `ops-ptc` rows' first round now carries the whole permitted `allowedTools` pool plus `ops_result` (the D1 sign-off item 1 decision, taken conservatively per #979 §3 so refs spilled this step stay consumable — and forced by the unified rule, since on-demand can always load `result`) with no `ops_load_sdk`. The first program can call the permitted pool directly; read-only sessions still exclude every write API, an empty cap stays at resident + `result`, `residentTools:false` drops the resident surface but not the tier, duplicated names register once, `global.dynamicTools:false` keeps its veto (the row publishes its legacy surface letter for letter) and a row that also turns its loader off keeps the tier. `PtcConfig.disclosurePolicy` stays optional because custom integrations keep the legacy on-demand default until #988; the builtin plan carries the parsed value. On-demand (default) behaviour is letter-identical; the settings-page entry is #989. +- 发布显式 PTC 全量披露档位(#987):`disclosurePolicy: 'full'` 的 `ptc` 与 `ops-ptc` 行首轮现在携带整个获准 `allowedTools` 池外加 `ops_result`(D1 签字项 1 的保守决定,依 #979 §3 让本步落盘的 ref 仍可消费——统一口径也逼出同一结论,因为按需档永远可加载 `result`),且无 `ops_load_sdk`。第一个程序即可直接调用获准池;只读会话仍排除全部写 API,空上限停在 常驻 + `result`,`residentTools:false` 丢常驻面但不丢档位,重复名只注册一次,`global.dynamicTools:false` 保持否决(该行逐字发布旧面),同时关掉 loader 的行保留档位。`PtcConfig.disclosurePolicy` 保持可选,因为自定义 integration 在 #988 之前维持旧 on-demand 缺省;内置计划携带解析值。按需档(缺省)行为逐字不变;设置页入口归 #989。 - Parse an explicit PTC disclosure policy without changing any legacy behaviour: the `ptc` and `ops-ptc` rows accept a YAML-only `disclosurePolicy` key, `'on-demand'` (default, the legacy progressive path) or `'full'`. Illegal values are errors naming the key and the received value or type, and non-PTC rows reject the key with a readable diagnostic. The choice survives read → save-unrelated-setting → read through the settings serializer, and no upgrade, settings save or `global.dynamicTools` flip ever writes or erases it. The D1 sign-off item on precedence is resolved conservatively: while `global.dynamicTools` is off the full tier is not applied — the row keeps its legacy resident/alwaysLoaded surface — and the saved choice re-applies when the switch returns on; the row-level `dynamicTools` keeps its loader-only meaning because the full tier publishes no loader. Publishing the full tier remains the D3 slice; no publication behaviour or settings UI changes (#986). - 解析显式 PTC 披露策略且不改变任何旧行为:`ptc` 与 `ops-ptc` 行接受 YAML-only 的 `disclosurePolicy` 键,取值 `'on-demand'`(缺省,即旧渐进路径)或 `'full'`。非法值报错点名键与收到的值或类型,非 PTC 行以可读诊断拒绝该键。选择经设置序列化器在「读取 → 保存无关设置 → 再读取」后不丢,升级、保存设置与 `global.dynamicTools` 翻转都不写入或抹掉它。D1 签字项中优先级问题按保守取向解决:`global.dynamicTools` 关闭期间全量档不生效——行保持旧 resident/alwaysLoaded 面——开关回开后已存选择重新生效;行级 `dynamicTools` 保持只管 loader 的含义,因为全量档不发布 loader。全量档的发布仍归 D3 切片;发布行为与设置 UI 均不变(#986)。 - Record the D1 supplementary audit for the legacy-config truth table: the custom PTC integration path rejects `alwaysLoaded` outside `allowedTools` (`src/consumer/options.ts`) while built-in rows accept it silently; the settings-page "discovery entry" copy is recorded verbatim next to the typed `ops_load_sdk` the PTC row actually publishes; a legal custom `profile: 'ptc'` integration is verified to be misdiagnosed `PUBLICATION_UNAVAILABLE: ops_result` (fix owned by #988); and one `ops_load_sdk` call is locked at 1-5 raw names (four calls minimum for the sixteen-name pool, resident lists uncapped). `test/ptc-config-truth.test.mjs` gains two assertions (13 checks); docs/design-decisions-20261119.md gains the matching append-only sections. No behaviour changes (#985). diff --git a/docs/configuration-reference.md b/docs/configuration-reference.md index 9152af9f..7485a91c 100644 --- a/docs/configuration-reference.md +++ b/docs/configuration-reference.md @@ -74,7 +74,7 @@ the row's status card, which sets it once when pressed. Generation and the manua | `hidePwsh` | true | Hide the official Windows shell while Bash is available; disabled entirely when `global.hidePwsh` is false | | `hideOfficialTools` | true | Hide the official file tools this mode replaces; disabled entirely when `global.hideOfficialTools` is false | | `dynamicTools` | true | Publish the progressive SDK loader `ops_load_sdk`; effective only while `global.dynamicTools` is also on (AND gate) | -| `disclosurePolicy` | `on-demand` | YAML-only explicit first-round disclosure tier: `on-demand` keeps the legacy progressive path (resident + loader); `full` publishes the whole `allowedTools` pool in the first round with no loader, once the D3 publication slice lands. Effective only while `global.dynamicTools` is on — the explicit opt-in never overrides the bundle switch; while it is off the row keeps its legacy resident/always-loaded surface and the saved choice is preserved. Rejected on non-PTC rows; illegal values are errors naming the key and the received value or type. No upgrade, settings save or switch flip ever writes this key for you | +| `disclosurePolicy` | `on-demand` | YAML-only explicit first-round disclosure tier: `on-demand` keeps the legacy progressive path (resident + loader); `full` publishes the whole `allowedTools` pool plus `result` in the first round with no loader (implemented; see the D3 addendum in docs/design-decisions-20261119.md). Effective only while `global.dynamicTools` is on — the explicit opt-in never overrides the bundle switch; while it is off the row keeps its legacy resident/always-loaded surface and the saved choice is preserved. Rejected on non-PTC rows; illegal values are errors naming the key and the received value or type. No upgrade, settings save or switch flip ever writes this key for you | | `batchingGuidance` | true for `ops-ptc`, false otherwise | Teach once at session start (logged history, system prompt untouched), then correct the shape of root `run_code` programs: `bypass`, `plan` and `shape` each latch once per turn and re-arm when the program complies; the notice rides the triggering result as `additionalContexts` and never blocks or rewrites the program. Text lives in the editable `prompts/batching-guidance.md`. YAML-only; an omitted `ops-ptc` section in a legacy whole-row profile override retains true; explicit false disables it | | `spillRetentionDays` | 30 | Host subprocess spill retention window; `0` reports only and never deletes | | `spillTotalBudgetMiB` | 256 | Total retained spill budget in MiB; `0` disables the budget | diff --git a/docs/design-decisions-20261119.md b/docs/design-decisions-20261119.md index 0805dec4..4d6f3983 100644 --- a/docs/design-decisions-20261119.md +++ b/docs/design-decisions-20261119.md @@ -35,13 +35,13 @@ present, real PTC presentation, the host's official `read`/`glob`/`grep` left vi | 3 | `ops-ptc: {dynamicTools: false}` | `ops_inspect_local_file` | none | no | same set as row 2 (one AND gate, two sides); only the scope differs — see note | | 4 | row 3 plus `ops-ptc: {alwaysLoaded: [stat]}` | `ops_inspect_local_file`, `ops_stat` | none | no | both bindings work in the first program | | 5 | `ops-ptc: {residentTools: false}` | `ops_load_sdk` | the sixteen-API pool below | yes | the loader is the only binding; `ops_inspect_local_file` is unreachable — `ops_load_sdk({tools:['inspect_local_file']})` answers `INVALID_ARGUMENT` | -| T | **target, not implemented**: an explicit full selection with `allowedTools: [grep, stat]` | `ops_grep`, `ops_stat` (per #979 §2) | — | no (per #979 §2) | every allowed API callable in the first program; read-only sessions still exclude write APIs | +| T | **implemented — see the D3 addendum**: an explicit full selection with `allowedTools: [grep, stat]` | `ops_grep`, `ops_stat` (per #979 §2) | — | no (per #979 §2) | every allowed API callable in the first program; read-only sessions still exclude write APIs | The default loadable pool (rows 1 and 5) is exactly: `ops_grep`, `ops_glob`, `ops_replace`, `ops_write_file`, `ops_edit_file`, `ops_diff`, `ops_stat`, `ops_copy`, `ops_move`, `ops_remove`, `ops_read_files`, `ops_read_table`, `ops_read_docx`, `ops_read_pdf`, `ops_batch`, `ops_result` — the sixteen non-resident -raw APIs (`PTC_LOADABLE_TOOLS`). Row T's pool would be `ops_grep`, `ops_stat` **plus +raw APIs (`PTC_LOADABLE_TOOLS`). Row T's pool is `ops_grep`, `ops_stat` **plus `ops_result`** (see sign-off item 1). **Row notes.** @@ -93,6 +93,7 @@ return r.status `dynamicTools:false` (rows 2–4) `ops_result` is **not published at all**. Decision needed for the full tier: must its first round publish `ops_result` so spilled refs stay consumable (#979 §3)? Current behaviour is locked; nothing here changes it. + *(Resolved by the D3 addendum: the full tier's first round publishes `ops_result`.)* 2. **The global switch versus an explicit full selection.** Today `global.dynamicTools` AND-gates every row's loader, including the PTC one. #979 §4 requires that upgrades, settings saves and the `global.dynamicTools` switch never pick full **for** the user. @@ -214,3 +215,52 @@ rejected as unknown), green after. `node test/run.mjs ptc-config-truth` 13/0/0, `node test/run.mjs resident-switch` 24/0/0 and `node test/run.mjs settings-rows` 14/0/0 unchanged on the new parsing path, worker `DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker` (sha256 2b5f988d). + + +## D3 addendum — the explicit full tier is published (2026-11-19) + +Card [#987](https://github.com/T-Auto/dsh-ops/issues/987), on top of the D2 addendum's +branch. This slice turns the parsed key into publication behaviour; nothing above +rewrites the five legacy rows, which stay asserted letter by letter. + +**Row T, implemented.** With `{ disclosurePolicy: 'full', allowedTools: [grep, stat] }` +the first round publishes `ops_grep`, `ops_stat`, `ops_inspect_local_file` (resident) +and `ops_result`, with no `ops_load_sdk` (#979 §2). The first program can call the +permitted pool directly (dual verification: the settled declaration set plus a real +`run_code` binding call through the worker); pool-outside business APIs stay +unavailable. The builtin plan slice now carries the parsed tier +(`src/presets/builtin.ts`), and the publication consumes it in +`src/publication/ptc.ts`. + +**Sign-off item 1, resolved conservatively per #979 §3.** The full tier's first round +publishes `ops_result`: a ref spilled this step must stay consumable, and the unified +acceptance rule forces the same conclusion — on-demand can always load `result` +(`loadable = [...allowedTools, 'result']`), so the loaded business set equals the full +first-round set only if full publishes it. The comparison is pinned by one check. + +**Edges, as shipped and asserted.** An empty cap keeps the tier at resident + +`result` (still no loader). Read-only still removes every mutating API from the +first round — switching tiers never widens permissions. `residentTools:false` drops +the resident surface (row 5) but not the explicit tier. Duplicated names collapse to +one registration. The bundle switch keeps its D2 veto end to end: while +`global.dynamicTools` is off an explicit full row publishes row 2 letter for letter; +the row-level `dynamicTools` keeps its loader-only meaning, so a full row that also +turns the loader off keeps its tier. + +**Not done here (by design).** Custom integrations keep the legacy default until D4, +and `PtcConfig.disclosurePolicy` stays optional for exactly that reason (making it +required would force touching the custom-integration plan slice this card does not +own; the builtin plan carries the value, an absent value means on-demand). The +settings-page entry is D5. + +**Verification.** `npm run build`, red first: `node test/run.mjs ptc-declaration-surface` +4 of 5 failed (every new row T assertion; the legacy loader-less check stayed green) +and `node test/run.mjs ptc-full-disclosure` failed row T, the comparison, read-only +and the empty pool before the suite's 120 s deadline. Green after: +ptc-declaration-surface 5/0/0, ptc-full-disclosure 8/0/0, and the regression locks +unchanged — ptc-config-truth 13/0/0, resident-switch 24/0/0, ptc-disclosure-policy +8/0/0, settings-rows 14/0/0; worker +`DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker` (sha256 2b5f988d). Environment +note, not a regression: `ptc.test.mjs`'s installed-runtime check asserts precedence +over `DSH_OPS_WORKER_BIN`, so it runs without that env override (it fails identically +on the base commit with the override set). diff --git "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" index f61d0d02..9662dd23 100644 --- "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" +++ "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" @@ -33,13 +33,13 @@ Linux、`danger-full-access`、native worker 就位、真实 PTC 呈现、宿主 | 3 | `ops-ptc: {dynamicTools: false}` | `ops_inspect_local_file` | 无 | 无 | 与第 2 行同一集合(同一个与门的两侧),差别只在作用域——见注 | | 4 | 第 3 行再加 `ops-ptc: {alwaysLoaded: [stat]}` | `ops_inspect_local_file`、`ops_stat` | 无 | 无 | 两个绑定首轮即可调用 | | 5 | `ops-ptc: {residentTools: false}` | `ops_load_sdk` | 下方十六个 API 的默认池 | 有 | loader 是唯一绑定;`ops_inspect_local_file` 整体不可达——`ops_load_sdk({tools:['inspect_local_file']})` 回 `INVALID_ARGUMENT` | -| T | **目标行,未实现**:显式选择全量且 `allowedTools: [grep, stat]` | `ops_grep`、`ops_stat`(依 #979 §2) | — | 无(依 #979 §2) | 获准 API 首轮全部可调;只读会话仍排除写 API | +| T | **已实现——见 D3 附录**:显式选择全量且 `allowedTools: [grep, stat]` | `ops_grep`、`ops_stat`(依 #979 §2) | — | 无(依 #979 §2) | 获准 API 首轮全部可调;只读会话仍排除写 API | 默认可加载池(第 1、5 行)恰为: `ops_grep`、`ops_glob`、`ops_replace`、`ops_write_file`、`ops_edit_file`、`ops_diff`、 `ops_stat`、`ops_copy`、`ops_move`、`ops_remove`、`ops_read_files`、`ops_read_table`、 `ops_read_docx`、`ops_read_pdf`、`ops_batch`、`ops_result`——十六个非常驻原始 API -(`PTC_LOADABLE_TOOLS`)。目标行 T 的池应为 `ops_grep`、`ops_stat` **外加 +(`PTC_LOADABLE_TOOLS`)。目标行 T 的池为 `ops_grep`、`ops_stat` **外加 `ops_result`**(见签字决定项 1)。 **逐行注释。** @@ -85,7 +85,7 @@ return r.status 可加载(现状由 `test/resident-switch.test.mjs` 的 "result exception remains loadable" 覆盖)。而 `dynamicTools:false`(第 2–4 行)下 `ops_result` **根本不发布**。全量档需要 决定:首轮是否发布 `ops_result`,让落盘的 ref 仍然可消费(#979 §3)?当前行为已锁定, - 本文不做任何改动。 + 本文不做任何改动。(已由 D3 附录解决:全量档首轮发布 `ops_result`。) 2. **总开关对显式全量选择的作用。** 今天 `global.dynamicTools` 与门控住每一行的 loader, PTC 行也不例外。#979 §4 要求升级、保存设置和 `global.dynamicTools` 开关都不得替用户 选择全量。需要决定:`global.dynamicTools:false` 是否也压制显式选择全量的行,还是显式 @@ -182,3 +182,40 @@ raw 与序列化两层都保持该键缺席。 `node test/run.mjs ptc-config-truth` 13/0/0、`node test/run.mjs resident-switch` 24/0/0、`node test/run.mjs settings-rows` 14/0/0 在新解析路径上不变,worker 用 `DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker`(sha256 2b5f988d)。 + + +## D3 附录——显式全量档已发布(2026-11-19) + +任务卡 [#987](https://github.com/T-Auto/dsh-ops/issues/987),叠在 D2 附录的分支之上。本切片 +把已解析的键变成发布行为;上文五行旧路径不被改写,仍由套件逐字断言。 + +**T 行,已实现。** `{ disclosurePolicy: 'full', allowedTools: [grep, stat] }` 的首轮发布 +`ops_grep`、`ops_stat`、`ops_inspect_local_file`(常驻)与 `ops_result`,且没有 +`ops_load_sdk`(#979 §2)。第一个程序即可直接调用获准池(双重验证:稳定声明集合 + +经 worker 的真实 `run_code` 绑定调用);池外业务 API 仍不可用。内置计划切片现在携带 +解析出的档位(`src/presets/builtin.ts`),发布在 `src/publication/ptc.ts` 消费它。 + +**签字决定项 1,按 #979 §3 取保守解。** 全量档首轮发布 `ops_result`:本步落盘的 ref +必须仍然可消费,且统一验收口径逼出同一结论——按需档永远可以加载 `result` +(`loadable = [...allowedTools, 'result']`),因此只有全量档也发布它,加载完毕的业务 +集合才等于全量首轮集合。该对比由一项检查钉住。 + +**边界,随实现一并断言。** 空上限使档位停在 常驻 + `result`(仍无 loader)。只读仍把 +一切变更 API 从首轮剔除——切换档位绝不放宽权限。`residentTools:false` 丢掉常驻面 +(第 5 行)但不丢显式档位。重复名收敛为一次注册。总开关保持 D2 的否决,端到端成立: +`global.dynamicTools` 关闭期间显式全量行逐字发布第 2 行;行级 `dynamicTools` 保持 +只管 loader 的含义,因此同时关掉 loader 的全量行保留自己的档位。 + +**本切片刻意不做。** 自定义 integration 在 D4 之前保持旧缺省, +`PtcConfig.disclosurePolicy` 也因此保持可选(改为必填会迫使本卡触碰不属于它的 +自定义 integration 计划切片;内置计划携带该值,缺省值即 on-demand)。设置页入口归 D5。 + +**验证。** `npm run build`,先红:`node test/run.mjs ptc-declaration-surface` +5 项中 4 项失败(全部新 T 行断言;旧 loader-less 检查保持绿), +`node test/run.mjs ptc-full-disclosure` 在套件 120 秒截止前 T 行、对比、只读与空池 +均失败。实现后全绿:ptc-declaration-surface 5/0/0、ptc-full-disclosure 8/0/0, +回归锁不变——ptc-config-truth 13/0/0、resident-switch 24/0/0、 +ptc-disclosure-policy 8/0/0、settings-rows 14/0/0;worker 用 +`DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker`(sha256 2b5f988d)。环境备注, +非回归:`ptc.test.mjs` 的安装包解析检查断言优先于 `DSH_OPS_WORKER_BIN`,因此该项 +在不设该环境变量时运行(在基准 commit 上设同样的变量,失败方式相同)。 diff --git "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" index 5a3835a8..c05bd510 100644 --- "a/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" +++ "b/docs/\351\205\215\347\275\256\345\217\202\350\200\203.md" @@ -47,7 +47,7 @@ OPS 沿用单行插件页:bundle 只插入一个 `dsh-ops` 行。内置预设 | `hidePwsh` | true | Bash 可用时隐藏 Windows 官方 shell;`global.hidePwsh` 为 false 时整体失效 | | `hideOfficialTools` | true | 隐藏本模式替代的官方文件工具;`global.hideOfficialTools` 为 false 时整体失效 | | `dynamicTools` | true | 发布渐进式 SDK 加载器 `ops_load_sdk`;与 `global.dynamicTools` 同时开启才生效(与门) | -| `disclosurePolicy` | `on-demand` | YAML-only 显式首轮披露档位:`on-demand` 保持旧渐进路径(常驻 + loader);`full` 在 D3 发布切片落地后首轮直接发布整个 `allowedTools` 池且无 loader。仅在 `global.dynamicTools` 开启时生效——显式 opt-in 不压过总开关;关闭期间该行保持旧 resident/alwaysLoaded 面,已保存的选择原样保留。非 PTC 行拒绝该键;非法值报错点名键与收到的值或类型。升级、保存设置与开关翻转都不会替你写入该键 | +| `disclosurePolicy` | `on-demand` | YAML-only 显式首轮披露档位:`on-demand` 保持旧渐进路径(常驻 + loader);`full` 首轮直接发布整个 `allowedTools` 池外加 `result`,且无 loader(已实现,详见 docs/设计决策-20261119.md 的 D3 附录)。仅在 `global.dynamicTools` 开启时生效——显式 opt-in 不压过总开关;关闭期间该行保持旧 resident/alwaysLoaded 面,已保存的选择原样保留。非 PTC 行拒绝该键;非法值报错点名键与收到的值或类型。升级、保存设置与开关翻转都不会替你写入该键 | | `batchingGuidance` | `ops-ptc` 为 true,其它模式为 false | 会话首轮注入一次完整教学(进入历史、不改系统提示词),之后按根 `run_code` 程序形状纠偏:`bypass`/`plan`/`shape` 三类每类每轮各锁一次、合规即解锁;提醒以 `additionalContexts` 附在触发调用结果上,不阻断也不改写程序。文案来自可编辑的 `prompts/batching-guidance.md`。仅 YAML 可配;旧 profile 整行覆盖配置且缺少 `ops-ptc` 节时仍默认 true,显式 false 可关闭 | | `spillRetentionDays` | 30 | 宿主 subprocess spill 保留窗口;`0` 只报告、永不删除 | | `spillTotalBudgetMiB` | 256 | 保留总字节预算(MiB);`0` 关闭预算 | diff --git a/src/presets/builtin.ts b/src/presets/builtin.ts index 3070f868..6b63fd13 100644 --- a/src/presets/builtin.ts +++ b/src/presets/builtin.ts @@ -51,7 +51,7 @@ export function applyResolvedMode(root: Context, config: ModeConfig, mode: ModeI shell: enabled && plan.shell && config.bash && process.platform === 'win32', presentation: plan.ptc ? 'ptc' : 'native', residentTools: enabled && config.residentTools, discovery: plan.ptc ? config.dynamicTools : options.discovery !== undefined && options.dynamic !== false && discovery.enabled, - native: discovery, ptc: { binaryPath: config.binaryPath, required: config.required, allowedTools: config.allowedTools, alwaysLoaded: config.alwaysLoaded }, + native: discovery, ptc: { binaryPath: config.binaryPath, required: config.required, allowedTools: config.allowedTools, alwaysLoaded: config.alwaysLoaded, disclosurePolicy: config.disclosurePolicy }, replaceOfficial: enabled && (config.hideOfficialTools || config.hidePwsh), replacementTargets: consumerTargets(config.hideOfficialTools && plan.nativeFiles, config.hidePwsh && config.bash && plan.shell), section: plan.section === null ? null : `mode-${plan.section}`, diff --git a/src/ptc/config.ts b/src/ptc/config.ts index cde03516..1153a60c 100644 --- a/src/ptc/config.ts +++ b/src/ptc/config.ts @@ -8,8 +8,8 @@ export const PTC_LOADABLE_TOOLS: readonly ApiName[] = [...FILE_TOOLS, ...PTC_TOO export type DisclosurePolicy = 'on-demand' | 'full' export interface PtcConfig { binaryPath?: string; required: boolean; allowedTools: ApiName[]; alwaysLoaded: ApiName[]; disclosurePolicy?: DisclosurePolicy } // parsePtcConfig always fills disclosurePolicy; consumer plan slices that have -// not been taught the tier yet (the builtin plan until D3, custom integrations -// until D4) may omit it and keep the legacy on-demand default. +// not been taught the tier yet (custom integrations until D4) may omit it and +// keep the legacy on-demand default. The builtin plan carries it since D3. export function parsePtcConfig(raw: unknown = {}): PtcConfig { if (!raw || typeof raw !== 'object' || Array.isArray(raw)) throw new Error('PTC config must be an object') const input = raw as Record diff --git a/src/publication/ptc.ts b/src/publication/ptc.ts index f9167c8e..0a82a513 100644 --- a/src/publication/ptc.ts +++ b/src/publication/ptc.ts @@ -74,10 +74,22 @@ export function mountPtc(root: Context, rawConfig: unknown, gate: PtcGate = { al let presentationRevision = 0 ctx.on('system-prompt/change', () => { ++presentationRevision }) let ownership: Awaited> | undefined + // D3 (#987, #979 §2): an explicit full tier carries the whole permitted + // allowedTools pool — plus `result`, the D1 sign-off item 1 decision, so a ref + // spilled this step never dead-ends (#979 §3; the unified acceptance rule + // also needs it: on-demand can always load `result`) — in the first round, + // with no loader. The bundle gate has already pressed an incompatible row + // back to on-demand (effectiveMode, sign-off item 2); the row-level + // dynamicTools keeps its loader-only meaning, so a full row that also turns + // the loader off keeps its explicit tier. + const full = config.disclosurePolicy === 'full' const resident = new Set(gate.resident === false ? [] : [...PTC_RESIDENT_TOOLS, ...config.alwaysLoaded]) // A loader-less scope can never select the allowed pool: registering it only // lets a host that renders registered definitions declare unreachable APIs. - const selectable: ApiName[] = gate.dynamic === false ? [] : [...config.allowedTools, 'result' as ApiName] + // The full tier is loader-less by design, so its pool must register: every + // member is callable this step. + const selectable: ApiName[] = gate.dynamic === false && !full ? [] : [...config.allowedTools, 'result' as ApiName] + const selectedFull = new Set(full ? selectable : []) const pool = new Set([...resident, ...selectable]) const owns = (agent: Agent): boolean => !stopped && gate.allows(agent) && Boolean(ownership?.owns(agent)) const permitted = (agent: Agent, api: ApiName | 'load_sdk'): boolean => { @@ -89,7 +101,7 @@ export function mountPtc(root: Context, rawConfig: unknown, gate: PtcGate = { al const loadable = [...new Set([...config.allowedTools, 'result'])] const allowedApis = (agent: Agent): ApiName[] => gate.dynamic === false ? [] : loadable.filter(api => permitted(agent, api)) const selected = (entry: Entry, api: ApiName | 'load_sdk'): boolean => api === 'load_sdk' - ? gate.dynamic !== false : resident.has(api) || entry.selected.has(api) + ? gate.dynamic !== false && !full : resident.has(api) || entry.selected.has(api) || selectedFull.has(api) const isPtc = async (agent: Agent, signal?: AbortSignal): Promise => { const assembly = await inspectPresentation(prompt, agent, signal) return assembly.tools.length === 1 && assembly.tools[0]?.name === 'run_code' @@ -246,7 +258,8 @@ export function mountPtc(root: Context, rawConfig: unknown, gate: PtcGate = { al } if (!owns(agent) || entries.get(agent) !== entry) return rotateFiles(entry) - if (gate.dynamic !== false) publish('load_sdk', sdkDefinition(loadable, execute(entry, 'load_sdk'))) + // The full tier is loader-less (#979 §2); the legacy tiers keep the loader gate. + if (gate.dynamic !== false && !full) publish('load_sdk', sdkDefinition(loadable, execute(entry, 'load_sdk'))) await Promise.all([...entry.disposers.map(publicationReady), ...[...entry.files.values()].map(file => file.lease.ready())]) }).catch(async error => { // Do not await ready here: this is its own rejection path. Preserve the diff --git a/test/ptc-declaration-surface.test.mjs b/test/ptc-declaration-surface.test.mjs index 656f396a..4f11b5f3 100644 --- a/test/ptc-declaration-surface.test.mjs +++ b/test/ptc-declaration-surface.test.mjs @@ -1,6 +1,6 @@ /** PTC declarations must follow the step admission, including loader-less scopes. */ import { createScope } from '@deepseek-ai/dsh-scope' -import { assert, test, report } from './lib/harness.mjs' +import { assert, test, report, waitFor } from './lib/harness.mjs' import { bootHost, shutdown } from './lib/host.mjs' import { prepareStep } from './lib/executor.mjs' import { mountPtc } from '../lib/publication/ptc.js' @@ -13,17 +13,17 @@ function declared(assembly) { return [...new Set([...sdk.matchAll(/^\s{2}(ops_[a-z_]+): \{/gm)].map(match => match[1]))].sort() } -async function setup(gate) { +async function setup(gate, ptc = {}, sandboxMode = 'danger-full-access') { const { ctx } = await bootHost({ withWorkingDirectory: true, toolsMode: 'ptc' }) const agents = [] ctx.provide('agents', { list: () => agents }) await ctx.plugin(inner => { inner.provide('ptcRuntime', { language: 'typescript' }) - inner.provide('sandboxPolicy', { resolve: () => ({ mode: 'danger-full-access', workspaceRoot: process.cwd() }) }) + inner.provide('sandboxPolicy', { resolve: () => ({ mode: sandboxMode, workspaceRoot: process.cwd() }) }) inner.provide('subprocess', { spawn() { throw new Error('declaration probe must never spawn') } }) }) - mountPtc(ctx, { binaryPath: process.execPath, required: true }, { allows: () => true, ...gate }) - const agent = { id: 'surface', session: { id: 'surface', mode: 'danger-full-access', header: { cwd: process.cwd() }, append() {}, read() { return [] } } } + mountPtc(ctx, { binaryPath: process.execPath, required: true, ...ptc }, { allows: () => true, ...gate }) + const agent = { id: 'surface', session: { id: 'surface', mode: sandboxMode, header: { cwd: process.cwd() }, append() {}, read() { return [] } } } agent.ctx = createScope(ctx, agent).ctx agents.push(agent) await ctx.serial('agent/created', { agent }) @@ -43,4 +43,60 @@ await test('a loader-less scope declares no selectable API even without a step t assert.deepEqual(admitted, []) } finally { await shutdown(ctx) } }) + +/** Poll the first-round SDK declaration until publication settles. */ +async function declaredNow(ctx, agent) { + const step = await prepareStep(ctx, agent) + try { return declared(step.assembly) } finally { step.dispose() } +} + +await test('an explicit full tier declares the whole permitted pool in the first round, loader-less', async () => { + // D1 truth-table row T (#987): the declaration half of the dual verification; + // the run_code binding half lives in the worker-backed ptc-full-disclosure suite. + const { ctx, agent } = await setup({ resident: true, dynamic: true }, { disclosurePolicy: 'full', allowedTools: ['grep', 'stat'] }) + try { + await waitFor(async () => (await declaredNow(ctx, agent)).includes('ops_grep'), + { what: 'ops_grep declared in the first round', timeoutMs: 10_000 }) + const admitted = await declaredNow(ctx, agent) + assert.deepEqual(admitted, ['ops_grep', 'ops_inspect_local_file', 'ops_result', 'ops_stat']) + assert.equal(admitted.includes('ops_load_sdk'), false, 'the full tier publishes no loader (#979 §2)') + assert.equal(admitted.includes('ops_glob'), false, 'a pool-outside business API stays undeclared') + } finally { await shutdown(ctx) } +}) + +await test('read-only keeps mutating APIs out of the full-tier declaration', async () => { + const { ctx, agent } = await setup({ resident: true, dynamic: true }, { disclosurePolicy: 'full', allowedTools: ['write_file', 'stat'] }, 'read-only') + try { + await waitFor(async () => (await declaredNow(ctx, agent)).includes('ops_stat'), + { what: 'ops_stat declared under read-only', timeoutMs: 10_000 }) + assert.deepEqual(await declaredNow(ctx, agent), ['ops_inspect_local_file', 'ops_result', 'ops_stat'], + 'the mutating member of the cap is never declared') + } finally { await shutdown(ctx) } +}) + +await test('an empty allowed pool declares only the resident surface plus result', async () => { + const { ctx, agent } = await setup({ resident: true, dynamic: true }, { disclosurePolicy: 'full', allowedTools: [] }) + try { + await waitFor(async () => (await declaredNow(ctx, agent)).includes('ops_result'), + { what: 'ops_result declared over an empty pool', timeoutMs: 10_000 }) + const admitted = await declaredNow(ctx, agent) + assert.deepEqual(admitted, ['ops_inspect_local_file', 'ops_result'], 'nothing else to publish, and still no loader') + assert.equal(admitted.includes('ops_load_sdk'), false) + } finally { await shutdown(ctx) } +}) + +await test('duplicate allowed names register exactly once in the full tier', async () => { + const { ctx, agent } = await setup({ resident: true, dynamic: true }, { disclosurePolicy: 'full', allowedTools: ['stat', 'stat', 'grep'] }) + try { + await waitFor(async () => (await declaredNow(ctx, agent)).includes('ops_grep'), + { what: 'ops_grep declared', timeoutMs: 10_000 }) + const step = await prepareStep(ctx, agent) + try { + const registered = ctx.get('tools').definitions(agent, step.toolSet).map(tool => tool.name) + assert.equal(registered.filter(name => name === 'ops_stat').length, 1, 'a duplicated name registers exactly once') + assert.deepEqual(declared(step.assembly), ['ops_grep', 'ops_inspect_local_file', 'ops_result', 'ops_stat']) + } finally { step.dispose() } + } finally { await shutdown(ctx) } +}) + report('ptc-declaration-surface') diff --git a/test/ptc-full-disclosure.test.mjs b/test/ptc-full-disclosure.test.mjs new file mode 100644 index 00000000..ea9e350f --- /dev/null +++ b/test/ptc-full-disclosure.test.mjs @@ -0,0 +1,214 @@ +/** The D3 publication slice: an explicit full tier actually publishes the pool. + * + * #979 splits the two-tier OPS-PTC work into slices; this suite locks the D3 + * slice (#987) end to end through the shipped merged row: with + * `disclosurePolicy: 'full'` the first round carries the whole permitted + * `allowedTools` pool plus `result` (the D1 sign-off item 1 decision, taken + * conservatively per #979 §3 so refs spilled this step stay consumable, and + * forced by the unified acceptance rule: on-demand can always load `result`, + * so the loaded set equals the full first round only if full publishes it) + * with no `ops_load_sdk`. Every check runs the real worker through the real + * PTC executor, and the dual verification the card requires: the settled + * first-round name set plus an actual run_code binding call. + */ +import { createScope, scopeOf, scopeParentOf } from '@deepseek-ai/dsh-scope' +import Subprocess from '@deepseek-ai/dsh-subprocess-local' +import Sandbox from '@deepseek-ai/dsh-sandbox-local' +import FileSystem from '@deepseek-ai/dsh-fs-local' +import NodePtc from '@deepseek-ai/dsh-ptc-runtime-node' +import fs from 'node:fs' +import path from 'node:path' +import { apply } from '../lib/index.js' +import { publicToolName } from '../lib/policy.js' +import { bootHost, shutdown } from './lib/host.mjs' +import { prepareStep, executeCall, selectExecutor } from './lib/executor.mjs' +import { assert, test, report, wait, waitFor, withTempDir } from './lib/harness.mjs' +import { WORKER, requireWorker } from './lib/boundary.mjs' + +requireWorker('ptc-full-disclosure') + +let sequence = 0 +const names = (ctx, agent) => ctx.get('tools').definitions(agent, agent.prepared?.toolSet).map(tool => tool.name).sort() +const opsNames = (ctx, agent) => names(ctx, agent).filter(name => name.startsWith('ops_')) +async function step(ctx, agent) { + ctx.emit('session/event', agent.session, { type: 'step/end', turn: 1, step: ++sequence }) + agent.prepared?.dispose(); agent.prepared = await prepareStep(ctx, agent) +} +async function scoped(parent, key, parentKey) { + let result + await parent.plugin({ inject: ['tools'], apply(inner) { + result = createScope(inner, key, parentKey ? { parent: parentKey } : undefined).ctx + } }) + return result +} +async function setup(cwd, switches, global = {}, sandboxMode = 'danger-full-access') { + const { ctx } = await bootHost({ withWorkingDirectory: true }) + await ctx.plugin(Subprocess); await ctx.plugin(Sandbox); await ctx.plugin(FileSystem); await ctx.plugin(NodePtc) + ctx.provide('sandboxPolicy', { resolve: ({ session } = {}) => ({ mode: session?.mode ?? sandboxMode, workspaceRoot: cwd, sessionId: session?.id }) }) + const presets = new Map(), agents = [] + ctx.provide('agents', { list: () => [...agents] }) + ctx.provide('agentPresets', { composedPreset: inner => presets.get(scopeParentOf(scopeOf(inner))) }) + const config = { global, discovery: { binaryPath: WORKER, required: true } } + for (const id of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) config[id] = { enabled: id === 'ops-ptc', + binaryPath: WORKER, required: true, ...(id === 'ops-ptc' ? switches : {}) } + await ctx.plugin({ name: 'full-disclosure-fixture', apply: inner => apply(inner, config) }) + const id = 'full-' + ++sequence, preset = await scoped(ctx, { preset: id }) + presets.set(scopeOf(preset), 'ops-ptc') + for (const name of ['read', 'glob', 'grep']) preset.tools.register({ name, description: name, + parameters: { type: 'object', properties: {} }, output: { schema: { type: 'object' }, render: () => [] }, execute: async () => ({}) }) + const agent = { id, session: { id, mode: sandboxMode, header: { cwd }, append() {}, read() { return [] } } } + agent.ctx = await scoped(preset, agent, scopeOf(preset)) + await agent.ctx.plugin({ inject: ['tools', 'systemPrompt'], apply: inner => { selectExecutor(inner, 'ptc') } }) + agents.push(agent) + await ctx.serial('agent/created', { agent }) + await step(ctx, agent) + return { ctx, agent } +} + +/** Boot one ops-ptc row, wait for its anchor, return the settled first-round set. */ +async function firstRound(switches, anchor, global = {}, sandboxMode = 'danger-full-access') { + return withTempDir('full-' + ++sequence, async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\nplain second line\n') + const { ctx, agent } = await setup(cwd, switches, global, sandboxMode) + try { + await waitFor(() => opsNames(ctx, agent).includes(anchor), { what: anchor + ' published' }) + await wait(100) + return opsNames(ctx, agent) + } finally { await shutdown(ctx) } + }) +} + +/** One real run_code program through the prepared step, as the loop issues it. */ +async function program(ctx, agent, code, label) { + const result = await executeCall(ctx, { agent, toolSet: agent.prepared.toolSet, name: 'run_code', + arguments: { description: label, code }, callId: 'full-' + ++sequence, signal: new AbortController().signal }) + assert.equal(result.isError, false, JSON.stringify(result)) + return result.value.result +} + +/** D1 truth-table row T: the explicit full selection, first round, no loader. */ +const ROW_T = ['ops_grep', 'ops_inspect_local_file', 'ops_result', 'ops_stat'] + +await test('row T: full + allowedTools=[grep,stat] publishes the whole pool in the first round, loader-less', () => withTempDir('full-row-t', async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\nplain second line\n') + const { ctx, agent } = await setup(cwd, { disclosurePolicy: 'full', allowedTools: ['grep', 'stat'] }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_grep'), { what: 'ops_grep published' }) + await wait(100) + assert.deepEqual(opsNames(ctx, agent), ROW_T) + assert.equal(opsNames(ctx, agent).includes('ops_load_sdk'), false, 'the full tier publishes no loader (#979 §2)') + // Dual verification: the binding works in the very first program, on a file + // with known content; pool-outside and loader bindings stay absent. + const value = await program(ctx, agent, "const stat = await tools.ops_stat({ path: 'fixture.txt' })\n" + + "const grep = await tools.ops_grep({ pattern: 'needle', path: '.' })\n" + + "return { statStatus: stat.status, grepStatus: grep.status, grepText: JSON.stringify(grep),\n" + + " loader: typeof tools.ops_load_sdk, glob: typeof tools.ops_glob }", 'Stat and grep the fixture') + assert.equal(value.statStatus, 'completed', JSON.stringify(value)) + assert.equal(value.grepStatus, 'completed', JSON.stringify(value)) + assert.match(value.grepText, /needle/, 'the real worker answered the grep call') + assert.match(value.grepText, /fixture\.txt/, 'the real worker answered the grep call') + assert.equal(value.loader, 'undefined', 'no loader binding in the full first round') + assert.equal(value.glob, 'undefined', 'a pool-outside business API is not callable') + } finally { await shutdown(ctx) } +})) + +await test('unified acceptance: on-demand after loading equals the full first round (loader excluded)', () => withTempDir('full-compare', async cwd => { + const fullSet = await firstRound({ disclosurePolicy: 'full', allowedTools: ['grep', 'stat'] }, 'ops_grep') + assert.deepEqual(fullSet, ROW_T) + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\n') + const { ctx, agent } = await setup(cwd, { allowedTools: ['grep', 'stat'] }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_load_sdk'), { what: 'ops_load_sdk published' }) + await wait(100) + assert.deepEqual(opsNames(ctx, agent), ['ops_inspect_local_file', 'ops_load_sdk'], 'on-demand starts at inspect + loader before any load') + const value = await program(ctx, agent, "const picked = await tools.ops_load_sdk({ tools: ['grep', 'stat', 'result'] })\n" + + "return { picked: picked.status, earlyGrep: typeof tools.ops_grep }", 'Load the same business set') + assert.equal(value.picked, 'completed', JSON.stringify(value)) + assert.equal(value.earlyGrep, 'undefined', 'a loaded API is callable only in the next program (#979 §1)') + await step(ctx, agent) + await wait(100) + const loaded = opsNames(ctx, agent).filter(name => name !== 'ops_load_sdk') + assert.deepEqual(loaded, ROW_T, 'on-demand loaded set = full first-round set, excluding the loader itself') + assert.deepEqual(loaded, fullSet) + } finally { await shutdown(ctx) } +})) + +await test('read-only excludes write APIs from the full first round', () => withTempDir('full-readonly', async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\n') + const { ctx, agent } = await setup(cwd, { disclosurePolicy: 'full', allowedTools: ['write_file', 'stat'] }, {}, 'read-only') + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_stat'), { what: 'ops_stat published' }) + await wait(100) + assert.deepEqual(opsNames(ctx, agent), ['ops_inspect_local_file', 'ops_result', 'ops_stat'], 'the mutating member of the cap is never published') + const value = await program(ctx, agent, "const stat = await tools.ops_stat({ path: 'fixture.txt' })\n" + + "return { statStatus: stat.status, writer: typeof tools.ops_write_file }", 'Stat under read-only') + assert.equal(value.statStatus, 'completed', JSON.stringify(value)) + assert.equal(value.writer, 'undefined', 'switching tiers never widens permissions (#979 §2)') + } finally { await shutdown(ctx) } +})) + +await test('an empty allowed pool keeps the full tier at the resident surface plus result', () => withTempDir('full-empty', async cwd => { + const { ctx, agent } = await setup(cwd, { disclosurePolicy: 'full', allowedTools: [] }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_inspect_local_file'), { what: 'ops_inspect_local_file published' }) + await wait(100) + assert.deepEqual(opsNames(ctx, agent), ['ops_inspect_local_file', 'ops_result'], 'nothing else to publish, and still no loader') + const value = await program(ctx, agent, "return { result: typeof tools.ops_result, grep: typeof tools.ops_grep }", 'Probe the empty pool') + assert.equal(value.result, 'function', 'result stays consumable so spilled refs never dead-end (#979 §3)') + assert.equal(value.grep, 'undefined') + } finally { await shutdown(ctx) } +})) + +await test('residentTools:false drops the resident set but keeps the explicit full tier', () => withTempDir('full-resident-off', async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\n') + const { ctx, agent } = await setup(cwd, { disclosurePolicy: 'full', allowedTools: ['grep', 'stat'], residentTools: false }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_grep'), { what: 'ops_grep published' }) + await wait(100) + assert.deepEqual(opsNames(ctx, agent), ['ops_grep', 'ops_result', 'ops_stat'], 'the resident conflict never silences the explicit tier') + const value = await program(ctx, agent, "const stat = await tools.ops_stat({ path: 'fixture.txt' })\n" + + "return { statStatus: stat.status, inspect: typeof tools.ops_inspect_local_file }", 'Stat with residents off') + assert.equal(value.statStatus, 'completed', JSON.stringify(value)) + assert.equal(value.inspect, 'undefined', 'residentTools:false still drops the resident surface') + } finally { await shutdown(ctx) } +})) + +await test('duplicate allowed names collapse to one publication each', () => withTempDir('full-duplicates', async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\n') + const { ctx, agent } = await setup(cwd, { disclosurePolicy: 'full', allowedTools: ['stat', 'stat', 'grep'] }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_grep'), { what: 'ops_grep published' }) + await wait(100) + assert.equal(opsNames(ctx, agent).filter(name => name === 'ops_stat').length, 1, 'a duplicated name registers exactly once') + assert.deepEqual(opsNames(ctx, agent), ROW_T) + const value = await program(ctx, agent, "const stat = await tools.ops_stat({ path: 'fixture.txt' })\nreturn { statStatus: stat.status }", 'Stat once') + assert.equal(value.statStatus, 'completed', JSON.stringify(value)) + } finally { await shutdown(ctx) } +})) + +await test('global dynamicTools:false presses an explicit full row back to the legacy surface', async () => { + // Sign-off item 2, end to end through the shipped row: the bundle switch keeps + // its veto, and the saved tier never overrides it (#979 §4). + const set = await firstRound({ disclosurePolicy: 'full', allowedTools: ['grep', 'stat'] }, 'ops_inspect_local_file', { dynamicTools: false }) + assert.deepEqual(set, ['ops_inspect_local_file'], 'row 2 of the D1 table, letter for letter') + assert.equal(set.includes('ops_grep'), false) + assert.equal(set.includes('ops_stat'), false) + assert.equal(set.includes('ops_load_sdk'), false) +}) + +await test('a full row that also turns its loader off keeps the explicit tier', () => withTempDir('full-loader-off', async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\n') + const { ctx, agent } = await setup(cwd, { disclosurePolicy: 'full', dynamicTools: false, allowedTools: ['grep', 'stat'] }) + try { + await waitFor(() => opsNames(ctx, agent).includes('ops_grep'), { what: 'ops_grep published' }) + await wait(100) + assert.deepEqual(opsNames(ctx, agent), ROW_T, 'row-level dynamicTools keeps its loader-only meaning') + assert.equal(opsNames(ctx, agent).includes('ops_load_sdk'), false) + const value = await program(ctx, agent, "const grep = await tools.ops_grep({ pattern: 'needle', path: '.' })\n" + + "return { grepStatus: grep.status, grepText: JSON.stringify(grep) }", 'Grep without a loader') + assert.equal(value.grepStatus, 'completed', JSON.stringify(value)) + assert.match(value.grepText, /needle/) + } finally { await shutdown(ctx) } +})) + +report('ptc-full-disclosure') From c68fe3d0e62e0e89b238c121899a30e60fd66231 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 14:13:01 +0800 Subject: [PATCH 8/9] test(ptc): drive the two-tier publication rule as a data-driven matrix --- docs/CHANGELOG.md | 2 + docs/design-decisions-20261119.md | 77 ++++ ...6\241\345\206\263\347\255\226-20261119.md" | 59 +++ test/ptc-two-tier-matrix.test.mjs | 357 ++++++++++++++++++ 4 files changed, 495 insertions(+) create mode 100644 test/ptc-two-tier-matrix.test.mjs diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ea2d24f3..10bc0aaa 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Turn the two-tier OPS-PTC publication rule into a data-driven acceptance matrix (#992): `test/ptc-two-tier-matrix.test.mjs` runs eight canonical rows (legacy default, the four legacy-off variants, on-demand first step, on-demand after loading, the explicit full first round, read-only, a narrow pool and a custom `profile: 'ptc'` integration) through one shared comparator across three views — the SDK text the model reads, the step admission the loop resolves and the `run_code` program bindings — plus a separately collected no-toolSet diagnostic view that is a registration superset, never an admission proof. Every row executes a real no-side-effect sentinel through the real worker (the read-only row's inspect runs under read-only) and write authority is asserted with a safe rejection naming the refused API. The drift property is itself asserted: removing any single expected name — or reading the diagnostic view where a step view is required — fails at least one cell with the missing/extra names printed, and the unified acceptance holds: the loaded business set equals the full first round on both publication paths. The official `read`/`glob`/`grep` trio is pinned as visible under both `hideOfficialTools` states while #961 is out of baseline, so its landing flips a reviewable pin. No publication behaviour changes. +- 把两档 OPS-PTC 发布规则做成数据驱动验收矩阵(#992):`test/ptc-two-tier-matrix.test.mjs` 用同一比较器跑八个规范行(旧缺省、四个旧关闭变体、按需首步、按需加载后、显式全量首轮、只读、窄池、自定义 `profile: 'ptc'` integration)的三个视角——模型读到的 SDK 文本、循环解析的当步准入、`run_code` 程序绑定——外加单列采集的无 toolSet 诊断视图(注册超集,绝非准入证据)。每行都经真实 worker 执行无副作用哨兵(只读行的 inspect 在只读沙箱下真跑),写权限以点名被拒 API 的安全拒绝断言。漂移性质本身被断言:删任一期望名或错读诊断视角,至少一格失败并打印缺失/多余名字;统一口径成立:按需加载完毕的业务集合与全量首轮在两条发布路径上一致。官方 `read`/`glob`/`grep` 三件在 #961 未入基线期间钉为两种 `hideOfficialTools` 状态下均可见,其合入将翻转可评审的钉子。发布行为不变。 - Publish the explicit full PTC disclosure tier (#987): with `disclosurePolicy: 'full'` the `ptc` and `ops-ptc` rows' first round now carries the whole permitted `allowedTools` pool plus `ops_result` (the D1 sign-off item 1 decision, taken conservatively per #979 §3 so refs spilled this step stay consumable — and forced by the unified rule, since on-demand can always load `result`) with no `ops_load_sdk`. The first program can call the permitted pool directly; read-only sessions still exclude every write API, an empty cap stays at resident + `result`, `residentTools:false` drops the resident surface but not the tier, duplicated names register once, `global.dynamicTools:false` keeps its veto (the row publishes its legacy surface letter for letter) and a row that also turns its loader off keeps the tier. `PtcConfig.disclosurePolicy` stays optional because custom integrations keep the legacy on-demand default until #988; the builtin plan carries the parsed value. On-demand (default) behaviour is letter-identical; the settings-page entry is #989. - 发布显式 PTC 全量披露档位(#987):`disclosurePolicy: 'full'` 的 `ptc` 与 `ops-ptc` 行首轮现在携带整个获准 `allowedTools` 池外加 `ops_result`(D1 签字项 1 的保守决定,依 #979 §3 让本步落盘的 ref 仍可消费——统一口径也逼出同一结论,因为按需档永远可加载 `result`),且无 `ops_load_sdk`。第一个程序即可直接调用获准池;只读会话仍排除全部写 API,空上限停在 常驻 + `result`,`residentTools:false` 丢常驻面但不丢档位,重复名只注册一次,`global.dynamicTools:false` 保持否决(该行逐字发布旧面),同时关掉 loader 的行保留档位。`PtcConfig.disclosurePolicy` 保持可选,因为自定义 integration 在 #988 之前维持旧 on-demand 缺省;内置计划携带解析值。按需档(缺省)行为逐字不变;设置页入口归 #989。 - Parse an explicit PTC disclosure policy without changing any legacy behaviour: the `ptc` and `ops-ptc` rows accept a YAML-only `disclosurePolicy` key, `'on-demand'` (default, the legacy progressive path) or `'full'`. Illegal values are errors naming the key and the received value or type, and non-PTC rows reject the key with a readable diagnostic. The choice survives read → save-unrelated-setting → read through the settings serializer, and no upgrade, settings save or `global.dynamicTools` flip ever writes or erases it. The D1 sign-off item on precedence is resolved conservatively: while `global.dynamicTools` is off the full tier is not applied — the row keeps its legacy resident/alwaysLoaded surface — and the saved choice re-applies when the switch returns on; the row-level `dynamicTools` keeps its loader-only meaning because the full tier publishes no loader. Publishing the full tier remains the D3 slice; no publication behaviour or settings UI changes (#986). diff --git a/docs/design-decisions-20261119.md b/docs/design-decisions-20261119.md index 4d6f3983..25770708 100644 --- a/docs/design-decisions-20261119.md +++ b/docs/design-decisions-20261119.md @@ -264,3 +264,80 @@ unchanged — ptc-config-truth 13/0/0, resident-switch 24/0/0, ptc-disclosure-po note, not a regression: `ptc.test.mjs`'s installed-runtime check asserts precedence over `DSH_OPS_WORKER_BIN`, so it runs without that env override (it fails identically on the base commit with the override set). + +## V3 addendum — the data-driven acceptance matrix (#992, 2026-11-19) + +Card [#992](https://github.com/T-Auto/dsh-ops/issues/992). This slice adds tests and +assertion helpers only: no publication behaviour changes. One new suite, +`test/ptc-two-tier-matrix.test.mjs` (16 checks), drives every configuration as data. + +**The matrix.** Eight canonical rows — legacy default, legacy off (the +`global.dynamicTools:false`, row-level `dynamicTools:false`, that row plus +`alwaysLoaded:[stat]`, and `residentTools:false` variants), on-demand first step, +on-demand after loading, the explicit full first round, read-only, a narrow pool, +and a custom `profile: 'ptc'` integration — each carrying its config inputs and the +exact business names it must publish. Every row is measured in three views with one +shared comparator: the SDK text the model reads (declared), the step admission the +loop resolves (admitted), and the `run_code` program bindings (callable). A fourth, +clearly separate view collects the no-toolSet diagnostic assembly: a host assembly +that omits the step tool set reads every registered definition, so it is a +registration superset, never an admission proof — the default row's diagnostic shows +all sixteen pool APIs plus inspect and the loader while its step admits exactly +inspect + loader. + +**Sentinels and safe rejections.** Every row executes a real no-side-effect sentinel +through the real worker (`ops_inspect_local_file`, `ops_stat`, or `ops_grep` over a +fixture file); the read-only row's inspect runs with the sandbox in read-only mode. +Write authority is asserted with a safe rejection: under read-only, +`ops_load_sdk({tools:['write_file']})` answers `INVALID_ARGUMENT` naming the refused +API with `effectState: 'none'` — no worker spawn, no side effect — and the +`ops_write_file` binding stays absent in all three step views while remaining visible +in the registration diagnostic. + +**The drift property is itself asserted.** Removing any single expected name from a +row's collected set fails exactly its cell and prints the culprit; inventing a name +fails as extra; and reading the diagnostic registry view where a step view is +required fails with the pool named. Demonstrated live for the PR: corrupting the +global-off row's expected set (as if the dynamic gate had been removed) fails with +`global-off:first/declared(ops) drifted: missing [ops_load_sdk]`, and restoring the +row returns the suite to green. + +**Unified acceptance.** With the same permission tier, the on-demand loaded business +set equals the full first round on both publication paths (preset and custom +integration) — the load set must include `result`, forced by the D3 sign-off item 1 +rule, so the comparison is exact including `ops_result`. A loaded API becomes +callable only in the next program; an unloaded or permission-revoked one never +pretends to be executable (`typeof` stays `'undefined'` in every row's probe). + +**#961 pinning.** [#961](https://github.com/T-Auto/dsh-ops/issues/961) (hide replaced +official file tools under PTC presets) is not in this baseline, so the matrix pins +today's truth: the official `read`/`glob`/`grep` trio is declared (in the `: ` +SDK shape, next to the `: {` ops shape) and admitted beside the ops surface under +both `global.hideOfficialTools` states — the switch is inert for PTC rows today, and +the `official-swap-off` row records it letter for letter. When #961 lands, these pins +go red for its on state and must be updated together with it; they exist so that flip +is a reviewable diff, not a silent drift. + +**Platform note.** The matrix is measured on Linux with the real worker, no Windows +simulation; POSIX rows keep the host shell, so `ops_bash` is absent from every +expected set. The Windows surface keeps its own suites on real Windows machines. + +**Current-state facts recorded along the way.** The custom integration path admits +its own `ops_status` diagnostics reader — admitted and callable beside the pool, but +never declared (it has no SDK projection) and not part of the two-tier kernel, so the +unified comparison excludes it. The loader description carries the configured cap +(`Configured raw names (current authority may narrow this pool): …`), not the +sandbox-narrowed one; a narrow pool narrows it correctly. The directory +recommendation text disappears once the configured pool is exhausted. Byte budgets +are V2 and real-model rounds are V5; the loader-less declaration half owned by +`test/ptc-declaration-surface.test.mjs` belongs to P5 and is untouched. + +**Verification.** `npm run build`; red first (the honest sequence): the first run +failed 5 rows on boot mechanics and 3 on wrong expectations — every correction +matched shipped behaviour or fixed the suite's own probe (the raw-name `tools.glob` +vs `ops_glob` binding typo, the lossless-JSON sentinel, the loader-less host set). +Green after: `DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker` (sha256 2b5f988d) +`node test/run.mjs ptc-two-tier-matrix` → all 16 checks passed in ~11 s; regressions +unchanged: ptc-full-disclosure 8/0/0, ptc-declaration-surface 5/0/0, +ptc-config-truth 13/0/0, resident-switch 24/0/0; `npm run check:size` clean. + diff --git "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" index 9662dd23..6ebfb9ea 100644 --- "a/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" +++ "b/docs/\350\256\276\350\256\241\345\206\263\347\255\226-20261119.md" @@ -219,3 +219,62 @@ ptc-disclosure-policy 8/0/0、settings-rows 14/0/0;worker 用 `DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker`(sha256 2b5f988d)。环境备注, 非回归:`ptc.test.mjs` 的安装包解析检查断言优先于 `DSH_OPS_WORKER_BIN`,因此该项 在不设该环境变量时运行(在基准 commit 上设同样的变量,失败方式相同)。 + +## V3 附录——数据驱动验收矩阵(#992,2026-11-19) + +卡 [#992](https://github.com/T-Auto/dsh-ops/issues/992)。本切片只加测试与断言 +helper,发布行为不变。新套件 `test/ptc-two-tier-matrix.test.mjs`(16 项检查)把 +每行配置当作数据驱动。 + +**矩阵。** 八个规范行——旧缺省、旧关闭(`global.dynamicTools:false`、行级 +`dynamicTools:false`、该行加 `alwaysLoaded:[stat]`、`residentTools:false` 四个变体)、 +按需首步、按需加载后、显式全量首轮、只读、窄池、自定义 `profile: 'ptc'` +integration——每行写明输入与必须发布的精确业务名。每行用同一比较器测三个视角: +模型读到的 SDK 文本(声明)、循环解析的当步准入(准入)、`run_code` 程序绑定 +(可调)。第四个明确分列的视角采集无 toolSet 的诊断装配:省略当步工具集的宿主 +装配读出全部已注册定义,它是注册超集,绝非准入证据——缺省行的诊断视图显示十六 +个池 API 加 inspect 与 loader,而其当步只准入 inspect + loader。 + +**哨兵与安全拒绝。** 每行都通过真实 worker 执行一个无副作用哨兵(对固定文件跑 +`ops_inspect_local_file`、`ops_stat` 或 `ops_grep`);只读行的 inspect 在只读沙箱下 +真跑。写权限用安全拒绝断言:只读下 `ops_load_sdk({tools:['write_file']})` 回 +`INVALID_ARGUMENT` 并点名被拒 API,`effectState: 'none'`——不 spawn worker、无副作用 +——且 `ops_write_file` 绑定在三个当步视角全部缺席,仅在注册诊断视图中可见。 + +**漂移性质本身被断言。** 从任一行已采集的集合里删掉任何一个期望名,恰好那一格 +失败并打印缺失名;凭空捏造名以多余报红;在该读当步视角处读注册诊断视角,则以池 +名报红。为 PR 现场演示过:把 global-off 行的期望集改坏(如同动态门被删), +失败信息为 `global-off:first/declared(ops) drifted: missing [ops_load_sdk]`, +还原后套件恢复全绿。 + +**统一口径。** 同权限档下,按需加载完毕的业务集合与全量首轮在两条发布路径 +(预设与自定义 integration)上都一致——加载集必须包含 `result`(D3 签字项 1 规则 +逼出),因此比较含 `ops_result` 逐名精确。加载的 API 只在下一个程序可调;未加载 +或被撤销权限的 API 绝不假装可执行(每行探针里 `typeof` 恒为 `'undefined'`)。 + +**#961 现状钉法。** [#961](https://github.com/T-Auto/dsh-ops/issues/961)(PTC 预设下 +隐藏被替代的官方文件工具)不在本基线,矩阵钉住今天的实况:官方 +`read`/`glob`/`grep` 三件以 `: ` 的 SDK 形状(并列于 ops 的 `: {` 形状)声明并 +准入,与 ops 面同在,且两种 `global.hideOfficialTools` 状态完全一致——该开关今天 +对 PTC 行无效,`official-swap-off` 行逐字记录。#961 合入时其开状态的钉子变红, +必须与它一并更新;钉子存在的意义就是让这次翻转成为可评审的 diff,而非静默漂移。 + +**平台说明。** 矩阵在 Linux 上以真实 worker 实测,不做 Windows 模拟;POSIX 行保留 +宿主 shell,所有期望集都不含 `ops_bash`。Windows 面由真实 Windows 机器上的独立 +套件负责。 + +**顺带钉住的现状事实。** 自定义 integration 路径额外准入自带的 `ops_status` 诊断 +读取器——与池并列地被准入且可调,但从不被声明(无 SDK 投影),不属两档内核, +统一比较将其排除。loader 描述携带配置上限(`Configured raw names (current +authority may narrow this pool): …`)而非沙箱收窄后的集合;窄池时正确收窄。配置池 +耗尽后目录推荐文本正确消失。字节分账归 V2、真实模型轮次归 V5; +`test/ptc-declaration-surface.test.mjs` 的无 loader 声明半边归 P5,未触碰。 + +**验证。** `npm run build`;先红(如实记录):首跑 5 行因引导机制失败、3 行因期望 +集错误——每处修正要么对齐已发布行为,要么修套件自身的探针(原始名 `tools.glob` +与 `ops_glob` 绑定笔误、无损 JSON 哨兵、无 loader 行的宿主集合)。后绿: +`DSH_OPS_WORKER_BIN=runtime/linux-x64/dsh-ops-worker`(sha256 2b5f988d) +`node test/run.mjs ptc-two-tier-matrix` → 16 项全过(约 11 秒);回归不变: +ptc-full-disclosure 8/0/0、ptc-declaration-surface 5/0/0、ptc-config-truth 13/0/0、 +resident-switch 24/0/0;`npm run check:size` 干净。 + diff --git a/test/ptc-two-tier-matrix.test.mjs b/test/ptc-two-tier-matrix.test.mjs new file mode 100644 index 00000000..a87857a5 --- /dev/null +++ b/test/ptc-two-tier-matrix.test.mjs @@ -0,0 +1,357 @@ +/** The #992 data-driven acceptance matrix for the two-tier publication rule. + * + * One kernel, three projections, one comparison. Every row is data: the config + * inputs and the exact business names the row must publish in each view — the + * SDK text the model reads (declared), the step admission the loop resolves + * (admitted), and the run_code program bindings (callable). A fourth, clearly + * separate view collects the no-toolSet diagnostic assembly: a host assembly + * that omits the step tool set reads every registered definition, so it is a + * registration superset, never an admission proof. All four go through one + * comparator that prints the missing and the extra names, and the drift + * property is itself asserted: removing any single expected name — or reading + * the diagnostic view where a step view is required — fails at least one cell + * with the culprit named. + * + * Every row executes a real sentinel through the real worker (the read-only + * row's inspect runs with the sandbox in read-only mode), and write authority + * is asserted with a safe rejection: ops_load_sdk never spawns the worker for + * a pool-outside or permission-denied name, it answers INVALID_ARGUMENT. + * + * The official read/glob/grep trio is pinned as *visible* next to the ops + * surface: #961 (hide replaced official file tools under PTC presets) is not + * in this baseline, and its hiding implementation is deliberately not + * simulated here. The official-swap-off row records that the shipped + * global.hideOfficialTools switch is inert for PTC rows today; when #961 lands + * these two rows flip together with it. + * + * Platform note: this matrix is measured on Linux with the real worker, no + * Windows simulation. POSIX rows keep the host shell, so ops_bash is out of + * every expected set; the Windows surface has its own suites on real Windows. + * Out of scope by card: byte accounting (V2) and real-model rounds (V5); the + * loader-less declaration half owned by test/ptc-declaration-surface.test.mjs + * belongs to P5 and is not touched here. + */ +import { createScope, scopeOf, scopeParentOf } from '@deepseek-ai/dsh-scope' +import Subprocess from '@deepseek-ai/dsh-subprocess-local' +import Sandbox from '@deepseek-ai/dsh-sandbox-local' +import FileSystem from '@deepseek-ai/dsh-fs-local' +import NodePtc from '@deepseek-ai/dsh-ptc-runtime-node' +import fs from 'node:fs' +import path from 'node:path' +import { apply } from '../lib/index.js' +import { integration } from '../lib/integration/index.js' +import { bootHost, shutdown } from './lib/host.mjs' +import { prepareStep, executeCall, selectExecutor } from './lib/executor.mjs' +import { assert, test, report, wait, waitFor, withTempDir } from './lib/harness.mjs' +import { WORKER, requireWorker } from './lib/boundary.mjs' + +requireWorker('ptc-two-tier-matrix') + +const OFFICIAL_TRIO = ['glob', 'grep', 'read'] +/** run_code is the executor's own tool: it is not part of the registry views this matrix compares. */ +const HOST_STEP_TOOLS = [...OFFICIAL_TRIO] +const I = 'ops_inspect_local_file' +const L = 'ops_load_sdk' +/** The sixteen-API default pool: every file API but the resident inspect, plus batch and result. */ +const POOL16 = ['ops_batch', 'ops_copy', 'ops_diff', 'ops_edit_file', 'ops_glob', 'ops_grep', 'ops_move', + 'ops_read_docx', 'ops_read_files', 'ops_read_pdf', 'ops_read_table', 'ops_remove', 'ops_replace', + 'ops_result', 'ops_stat', 'ops_write_file'] + +let sequence = 0 + +/** One comparison for every cell of the matrix: exact set equality, culprits named. */ +function expectCell(rowId, view, expected, actual) { + const missing = expected.filter(name => !actual.includes(name)) + const extra = actual.filter(name => !expected.includes(name)) + if (missing.length || extra.length) + throw new Error(`matrix cell ${rowId}/${view} drifted: missing [${missing.join(', ')}], extra [${extra.join(', ')}]`) +} + +const sdkTextOf = assembly => assembly.sections.find(section => section.name === 'tools:sdk')?.text ?? '' +/** The ops_* names one assembly declares in tools:sdk (the ": {" shape). */ +const declaredOps = assembly => [...new Set([...sdkTextOf(assembly).matchAll(/^ {2}(ops_[a-z_]+): \{/gm)].map(match => match[1]))].sort() +/** The official trio declared in tools:sdk (the ": " shape; #961 is not in this baseline). */ +const declaredOfficial = assembly => OFFICIAL_TRIO.filter(name => new RegExp(`^ {2}${name}: `, 'm').test(sdkTextOf(assembly))) +/** A host assembly without the step tool set reads every registered definition; never an admission proof. */ +const diagnosticOps = async (ctx, agent) => { + const assembly = await ctx.get('systemPrompt').assemble({ scope: agent }) + return [...new Set([...sdkTextOf(assembly).matchAll(/^ {2}(ops_[a-z_]+): \{/gm)].map(match => match[1]))].sort() +} +const admittedNames = (ctx, agent) => ctx.get('tools').definitions(agent, agent.prepared.toolSet).map(tool => tool.name) +const admittedOps = names => names.filter(name => name.startsWith('ops_')).sort() +const admittedHost = names => names.filter(name => !name.startsWith('ops_')).sort() + +async function scoped(parent, key, parentKey) { + let result + await parent.plugin({ inject: ['tools'], apply(inner) { + result = createScope(inner, key, parentKey ? { parent: parentKey } : undefined).ctx + } }) + return result +} + +/** Boot one ops-ptc preset row exactly as the shipped settings compose it. */ +async function bootPresetRow(cwd, row) { + const sandboxMode = row.sandboxMode ?? 'danger-full-access' + const { ctx } = await bootHost({ withWorkingDirectory: true }) + await ctx.plugin(Subprocess); await ctx.plugin(Sandbox); await ctx.plugin(FileSystem); await ctx.plugin(NodePtc) + ctx.provide('sandboxPolicy', { resolve: ({ session } = {}) => ({ mode: session?.mode ?? sandboxMode, workspaceRoot: cwd, sessionId: session?.id }) }) + const presets = new Map(), agents = [] + ctx.provide('agents', { list: () => [...agents] }) + ctx.provide('agentPresets', { composedPreset: inner => presets.get(scopeParentOf(scopeOf(inner))) }) + const config = { global: row.global ?? {}, discovery: { binaryPath: WORKER, required: true } } + for (const id of ['minimal', 'standard', 'ptc', 'ops-ptc', 'cordis']) config[id] = { enabled: id === 'ops-ptc', + binaryPath: WORKER, required: true, ...(id === 'ops-ptc' ? row.switches : {}) } + await ctx.plugin({ name: 'matrix-preset-fixture', apply: inner => apply(inner, config) }) + const id = 'matrix-' + row.id + '-' + ++sequence, preset = await scoped(ctx, { preset: id }) + presets.set(scopeOf(preset), 'ops-ptc') + for (const name of OFFICIAL_TRIO) preset.tools.register({ name, description: name, + parameters: { type: 'object', properties: {} }, output: { schema: { type: 'object' }, render: () => [] }, execute: async () => ({}) }) + const agent = { id, session: { id, mode: sandboxMode, header: { cwd }, append() {}, read() { return [] } } } + agent.ctx = await scoped(preset, agent, scopeOf(preset)) + await agent.ctx.plugin({ inject: ['tools', 'systemPrompt'], apply: inner => { selectExecutor(inner, 'ptc') } }) + agents.push(agent) + await ctx.serial('agent/created', { agent }) + await advanceStep(ctx, agent) + return { ctx, agent } +} + +/** Boot one custom integration row: the plan slice integrations compose themselves (#988 keeps it on-demand). */ +async function bootIntegrationRow(cwd, row) { + const sandboxMode = row.sandboxMode ?? 'danger-full-access' + const { ctx } = await bootHost({ withWorkingDirectory: true }) + await ctx.plugin(Subprocess); await ctx.plugin(Sandbox); await ctx.plugin(FileSystem); await ctx.plugin(NodePtc) + ctx.provide('sandboxPolicy', { resolve: ({ session } = {}) => ({ mode: session?.mode ?? sandboxMode, workspaceRoot: cwd, sessionId: session?.id }) }) + ctx.provide('agentPresets', { composedPreset: () => undefined }) + const agents = [] + ctx.provide('agents', { list: () => [...agents] }) + await ctx.plugin(integration({ preset: row.preset, profile: 'ptc', binaryPath: WORKER, required: true, ...row.switches })) + const id = row.preset + '-' + ++sequence + const agent = { id, session: { id, mode: sandboxMode, header: { cwd, agentPreset: row.preset }, append() {}, read() { return [] } } } + await ctx.plugin({ inject: ['tools'], apply(inner) { agent.ctx = createScope(inner, agent).ctx } }) + await agent.ctx.plugin({ inject: ['tools', 'systemPrompt'], apply: inner => { selectExecutor(inner, 'ptc') } }) + agents.push(agent) + await ctx.serial('agent/created', { agent }) + await advanceStep(ctx, agent) + return { ctx, agent } +} + +async function advanceStep(ctx, agent) { + ctx.emit('session/event', agent.session, { type: 'step/end', turn: ++sequence, step: ++sequence }) + agent.prepared?.dispose() + agent.prepared = await prepareStep(ctx, agent) +} + +/** One real run_code program through the prepared step, as the loop issues it. */ +async function program(ctx, agent, code, label) { + const result = await executeCall(ctx, { agent, toolSet: agent.prepared.toolSet, name: 'run_code', + arguments: { description: label, code }, callId: 'matrix-' + ++sequence, signal: new AbortController().signal }) + assert.equal(result.isError, false, JSON.stringify(result)) + return result.value.result +} + +/** The no-side-effect sentinel each row really executes through the worker. */ +function sentinelCode(spec) { + if (spec === 'inspect') return { code: "const sentinel = await tools.ops_inspect_local_file({ path: 'fixture.txt', limit: 2 })", + checks: value => { assert.equal(value.sentinelStatus, 'completed', JSON.stringify(value)) + assert.equal(value.sentinelText, true, 'the real worker answered the inspect call') } } + if (spec === 'grep') return { code: "const sentinel = await tools.ops_grep({ pattern: 'needle', path: '.' })", + checks: value => { assert.equal(value.sentinelStatus, 'completed', JSON.stringify(value)) + assert.equal(value.sentinelText, true, 'the real worker answered the grep call') } } + if (spec === 'stat') return { code: "const sentinel = await tools.ops_stat({ path: 'fixture.txt' })", + checks: value => { assert.equal(value.sentinelStatus, 'completed', JSON.stringify(value)) } } + if (spec === 'none') return { code: 'const sentinel = null', checks: () => {} } + throw new Error('unknown sentinel: ' + spec) +} + +/** Collect the four views of one settled state and drive them all through expectCell. */ +async function collectViews(ctx, agent, row, phase, evidence) { + const spec = row.expect[phase] + const names = admittedNames(ctx, agent) + const assembly = agent.prepared.assembly + expectCell(row.id + ':' + phase, 'declared(ops)', spec.ops, declaredOps(assembly)) + expectCell(row.id + ':' + phase, 'declared(official)', spec.official, declaredOfficial(assembly)) + expectCell(row.id + ':' + phase, 'admitted(ops)', [...spec.ops, ...(spec.admittedExtra ?? [])], admittedOps(names)) + expectCell(row.id + ':' + phase, 'admitted(host)', spec.host, admittedHost(names)) + // The diagnostic view is registration state: a load changes selection, not registration. + if (spec.diagnostic) expectCell(row.id + ':' + phase, 'diagnostic(ops)', spec.diagnostic, await diagnosticOps(ctx, agent)) + assert.equal(assembly.sections.some(section => section.text.includes('For on-demand APIs, use tools.ops_load_sdk')), + spec.directory, `${row.id}:${phase} directory recommendation text`) + if (spec.loaderDescription) { + const loader = ctx.get('tools').definitions(agent, agent.prepared.toolSet).find(tool => tool.name === L) + for (const word of spec.loaderDescription.includes) assert.ok(loader?.description.includes(word), `${row.id} loader description names ${word}`) + for (const word of spec.loaderDescription.excludes) assert.equal(loader?.description.includes(word), false, `${row.id} loader description must stay narrowed (${word})`) + } + // The callable view: one real program reports typeof for every probed name plus the sentinel. + const probed = [...new Set([...spec.callable, ...spec.absent])] + const sentinel = sentinelCode(row.sentinel[phase]) + const noSentinel = row.sentinel[phase] === 'none' + const reject = row.reject && phase === 'first' + ? `const rejection = await tools.ops_load_sdk({ tools: ${JSON.stringify([row.reject.tool])} })` : '' + const value = await program(ctx, agent, sentinel.code + '\n' + reject + '\nreturn { ' + + probed.map(name => `${name}: typeof tools.${name}, `).join('') + + (noSentinel ? '' : 'sentinelStatus: sentinel?.status, sentinelText: JSON.stringify(sentinel).includes(\'needle\'), ') + + (reject ? 'rejectStatus: rejection?.status, rejectCode: rejection?.error?.code, ' + + 'rejectMessage: rejection?.error?.message, rejectEffect: rejection?.error?.effectState, ' : '') + + ' }', 'Matrix probe ' + row.id + ':' + phase) + const callable = probed.filter(name => value[name] === 'function').sort() + expectCell(row.id + ':' + phase, 'callable', spec.callable, callable) + for (const name of spec.absent) assert.equal(value[name], 'undefined', `${row.id}:${phase} ${name} must not pretend to be executable`) + if (!noSentinel) sentinel.checks(value) + if (reject) { + assert.equal(value.rejectStatus, 'error', 'the safe rejection is a returned value, not a throw') + assert.equal(value.rejectCode, row.reject.code, JSON.stringify(value)) + assert.ok(value.rejectMessage.includes(row.reject.tool), 'the rejection names the refused API: ' + value.rejectMessage) + assert.equal(value.rejectEffect, 'none', 'a refused load has no effect') + } + evidence[row.id] = { ...(evidence[row.id] ?? {}), [phase]: { ops: admittedOps(names), diagnostic: await diagnosticOps(ctx, agent), + official: declaredOfficial(assembly), callable } } +} + +/** Run one matrix row end to end and stash its collected views for the synthesis checks. */ +async function runRow(row, evidence) { + return withTempDir('matrix-' + row.id, async cwd => { + fs.writeFileSync(path.join(cwd, 'fixture.txt'), 'needle in haystack\nplain second line\n') + const { ctx, agent } = row.kind === 'integration' ? await bootIntegrationRow(cwd, row) : await bootPresetRow(cwd, row) + try { + await waitFor(() => admittedOps(admittedNames(ctx, agent)).includes(row.anchor), { what: row.anchor + ' admitted', timeoutMs: 20_000 }) + await wait(120) + agent.prepared.dispose(); agent.prepared = await prepareStep(ctx, agent) + await collectViews(ctx, agent, row, 'first', evidence) + if (row.load) { + const value = await program(ctx, agent, `const receipt = await tools.ops_load_sdk({ tools: ${JSON.stringify(row.load)} })\n` + + `return { receiptStatus: receipt.status, early: typeof tools.ops_${row.load[0]} }`, 'Matrix load ' + row.id) + assert.equal(value.receiptStatus, 'completed', JSON.stringify(value)) + assert.equal(value.early, 'undefined', row.id + ': a loaded API is callable only in the next program') + await advanceStep(ctx, agent) + await wait(120) + await collectViews(ctx, agent, row, 'loaded', evidence) + } + } finally { await shutdown(ctx) } + }) +} + +// ---- The matrix rows. Eight canonical configurations (issue #992) with the +// legacy-off variants spelled out, each carrying inputs and expected names. ---- + +const ON_DEMAND_CAP = { ops: [I, L], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], + diagnostic: [I, L, 'ops_glob', 'ops_grep', 'ops_result'], callable: [I, L], + absent: ['ops_glob', 'ops_grep', 'ops_result', 'ops_stat', 'ops_write_file', 'ops_batch'], directory: true } + +const ROWS = [ + { id: 'legacy-default', anchor: L, sentinel: { first: 'inspect' }, switches: {}, expect: { first: { + ...ON_DEMAND_CAP, diagnostic: [I, L, ...POOL16] } } }, + // #961 is out of baseline: the shipped hideOfficialTools switch is inert for PTC + // rows, so its off state publishes the identical surface. #961 flips this row. + { id: 'official-swap-off', anchor: L, sentinel: { first: 'inspect' }, global: { hideOfficialTools: false }, + expect: { first: { ...ON_DEMAND_CAP, diagnostic: [I, L, ...POOL16] } } }, + { id: 'global-off', anchor: I, sentinel: { first: 'inspect' }, global: { dynamicTools: false }, expect: { first: { + ops: [I], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], diagnostic: [I], callable: [I], + absent: [L, ...POOL16], directory: false } } }, + { id: 'row-off', anchor: I, sentinel: { first: 'inspect' }, switches: { dynamicTools: false }, expect: { first: { + ops: [I], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], diagnostic: [I], callable: [I], + absent: [L, ...POOL16], directory: false } } }, + { id: 'row-off-always', anchor: 'ops_stat', sentinel: { first: 'stat' }, switches: { dynamicTools: false, alwaysLoaded: ['stat'] }, expect: { first: { + ops: [I, 'ops_stat'], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], diagnostic: [I, 'ops_stat'], + callable: [I, 'ops_stat'], absent: [L, ...POOL16.filter(name => name !== 'ops_stat')], directory: false } } }, + { id: 'resident-off', anchor: L, load: ['grep'], reject: { tool: 'inspect_local_file', code: 'INVALID_ARGUMENT' }, + sentinel: { first: 'none', loaded: 'grep' }, switches: { residentTools: false }, expect: { + first: { ops: [L], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], diagnostic: [L, ...POOL16], + callable: [L], absent: [I, ...POOL16], directory: true }, + loaded: { ops: [L, 'ops_grep'], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], + callable: [L, 'ops_grep'], absent: [I, 'ops_result', 'ops_stat', 'ops_glob'], directory: true } } }, + { id: 'on-demand-first', anchor: L, sentinel: { first: 'inspect' }, switches: { allowedTools: ['glob', 'grep'] }, + expect: { first: ON_DEMAND_CAP } }, + // The unified acceptance rule (#979 §3) forces result into the load set: the + // full first round publishes it, so the loaded business set equals it only + // when on-demand loads it explicitly, like any other pool member. + { id: 'on-demand-loaded', anchor: L, load: ['glob', 'grep', 'result'], sentinel: { first: 'inspect', loaded: 'grep' }, + switches: { allowedTools: ['glob', 'grep'] }, expect: { first: ON_DEMAND_CAP, + // The pool is exhausted after the load: the directory recommendation + // correctly disappears when no allowed name is left to load. + loaded: { ops: [I, L, 'ops_glob', 'ops_grep', 'ops_result'], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], + callable: [I, L, 'ops_glob', 'ops_grep', 'ops_result'], absent: ['ops_stat', 'ops_write_file', 'ops_batch'], directory: false } } }, + { id: 'full-direct', anchor: 'ops_glob', sentinel: { first: 'grep' }, + switches: { disclosurePolicy: 'full', allowedTools: ['glob', 'grep'] }, expect: { first: { + ops: [I, 'ops_glob', 'ops_grep', 'ops_result'], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], + diagnostic: [I, 'ops_glob', 'ops_grep', 'ops_result'], callable: [I, 'ops_glob', 'ops_grep', 'ops_result'], + absent: [L, 'ops_stat', 'ops_write_file', 'ops_batch'], directory: false } } }, + { id: 'read-only', anchor: L, sandboxMode: 'read-only', reject: { tool: 'write_file', code: 'INVALID_ARGUMENT' }, + sentinel: { first: 'inspect' }, switches: { allowedTools: ['write_file', 'grep'] }, expect: { first: { + ops: [I, L], host: [...HOST_STEP_TOOLS], official: [...OFFICIAL_TRIO], + diagnostic: [I, L, 'ops_grep', 'ops_result', 'ops_write_file'], callable: [I, L], + absent: ['ops_write_file', 'ops_grep', 'ops_result', 'ops_stat'], directory: true, + loaderDescription: { includes: ['write_file, grep, result'], excludes: [] } } } }, + { id: 'narrow-pool', anchor: L, reject: { tool: 'stat', code: 'INVALID_ARGUMENT' }, sentinel: { first: 'inspect' }, + switches: { allowedTools: ['grep'] }, expect: { first: { ...ON_DEMAND_CAP, + diagnostic: [I, L, 'ops_grep', 'ops_result'], + loaderDescription: { includes: ['grep, result'], excludes: ['stat', 'glob'] } } } }, + // The custom integration path also admits its own ops_status diagnostics reader, + // which has no SDK projection: it is admitted and callable but never declared. + { id: 'custom-preset', kind: 'integration', preset: 'matrix-custom', anchor: L, load: ['glob', 'grep', 'result'], + sentinel: { first: 'inspect', loaded: 'grep' }, switches: { allowedTools: ['glob', 'grep'] }, expect: { + first: { ops: [I, L], admittedExtra: ['ops_status'], host: [], official: [], diagnostic: [I, L, 'ops_glob', 'ops_grep', 'ops_result'], + callable: [I, L, 'ops_status'], absent: ['ops_glob', 'ops_grep', 'ops_result', 'ops_stat'], directory: true }, + loaded: { ops: [I, L, 'ops_glob', 'ops_grep', 'ops_result'], admittedExtra: ['ops_status'], host: [], official: [], + callable: [I, L, 'ops_glob', 'ops_grep', 'ops_result', 'ops_status'], absent: ['ops_stat', 'ops_write_file'], directory: false } } }, +] + +/** Collected views of every row, reused by the synthesis checks below. */ +const evidence = {} + +for (const row of ROWS) { + await test('matrix row ' + row.id + ': three views and the diagnostic view agree with the row', () => runRow(row, evidence)) +} + +await test('unified acceptance: the on-demand loaded business set equals the full first round on both publication paths', () => { + const business = evidence['on-demand-loaded'].loaded.ops.filter(name => name !== L) + const full = evidence['full-direct'].first.ops + // ops_status is the custom integration's own diagnostics reader, admitted + // beside the pool; it is not a member of the two-tier publication kernel. + const custom = evidence['custom-preset'].loaded.ops.filter(name => name !== L && name !== 'ops_status') + expectCell('unified-acceptance', 'preset-loaded-vs-full', full, business) + expectCell('unified-acceptance', 'custom-loaded-vs-full', full, custom) +}) + +await test('drift property: removing any one expected name fails exactly its cell and prints the culprit', () => { + const stepOps = evidence['on-demand-first'].first.ops + for (const name of stepOps) { + const corrupted = stepOps.filter(candidate => candidate !== name) + assert.throws(() => expectCell('on-demand-first', 'declared(ops)', corrupted, stepOps), + new RegExp('missing \\[.*' + name), name + ' removal must fail with the name printed') + } + assert.throws(() => expectCell('on-demand-first', 'declared(ops)', stepOps, [...stepOps, 'ops_nope']), + /extra \[.*ops_nope/, 'an invented name must fail as extra with the name printed') + assert.throws(() => expectCell('on-demand-first', 'declared(ops)', [...stepOps, 'ops_nope'], stepOps), + /missing \[.*ops_nope/, 'a lost expected name must fail as missing with the name printed') + const callable = evidence['on-demand-first'].first.callable + assert.throws(() => expectCell('on-demand-first', 'callable', callable.filter(name => name !== L), callable), + new RegExp('missing \\[.*' + L), 'deleting the loader admission fails the callable cell') +}) + +await test('drift property: reading the diagnostic registry view cannot satisfy a step cell', () => { + // The diagnostic assembly registers the whole pool while the step admits + // only inspect + loader, so a wrong-view read must fail with the pool named. + const step = evidence['on-demand-first'].first.ops + const diagnostic = evidence['legacy-default'].first.diagnostic + assert.throws(() => expectCell('on-demand-first', 'declared(ops)', step, diagnostic), + /extra \[.*ops_grep/, 'the pool registration must be reported as extra names') + assert.throws(() => expectCell('legacy-default', 'diagnostic(ops)', diagnostic, step), + /missing \[.*ops_grep/, 'the step view must be reported as missing names') +}) + +await test('the official trio stays visible next to the ops surface while #961 is out of baseline', () => { + // #961 (hide replaced official file tools under PTC presets) is not in this + // baseline, so today the trio is declared and admitted next to the ops + // surface under both hideOfficialTools states — the switch is inert for PTC + // rows. When #961 lands, these pins go red for its on state and must be + // updated together with it; they exist so that flip is a reviewable diff. + for (const id of ['legacy-default', 'official-swap-off']) { + const first = evidence[id].first + expectCell(id, 'declared(official)', OFFICIAL_TRIO, OFFICIAL_TRIO) + assert.deepEqual(first.ops, [I, L], id + ' publishes the resident surface plus the loader') + } + assert.deepEqual(evidence['legacy-default'].first.ops, evidence['official-swap-off'].first.ops, + 'the shipped hideOfficialTools switch is inert for PTC rows today') +}) + +report('ptc-two-tier-matrix') From 8ce07c07e1ff57fecbd01a2138e625d93e15e257 Mon Sep 17 00:00:00 2001 From: Thatgfsj <171755367+Thatgfsj@users.noreply.github.com> Date: Thu, 19 Nov 2026 15:54:58 +0800 Subject: [PATCH 9/9] bench(v5): measure the Flash disclosure tiers with the full denominator --- bench/lib/tracks.mjs | 4 +- bench/v5-collect.mjs | 301 +++++++++++++ bench/v5-run.mjs | 486 +++++++++++++++++++++ docs/CHANGELOG.md | 2 + "docs/Flash\345\257\271\347\205\247-V5.md" | 154 +++++++ docs/flash-comparison-v5.md | 172 ++++++++ 6 files changed, 1117 insertions(+), 2 deletions(-) create mode 100644 bench/v5-collect.mjs create mode 100644 bench/v5-run.mjs create mode 100644 "docs/Flash\345\257\271\347\205\247-V5.md" create mode 100644 docs/flash-comparison-v5.md diff --git a/bench/lib/tracks.mjs b/bench/lib/tracks.mjs index 88de8af3..a0e19adb 100644 --- a/bench/lib/tracks.mjs +++ b/bench/lib/tracks.mjs @@ -65,7 +65,7 @@ const stepKey = (step) => step.api + ' ' + JSON.stringify(step.args) * @param {(fact: string) => void} add - record one failed fact. * @returns {number} how many legitimate waits the trajectory spent. */ -function scanReissues(steps, add) { +export function scanReissues(steps, add) { let waits = 0 const previous = new Map() for (const step of steps) { @@ -93,7 +93,7 @@ function scanReissues(steps, add) { * @param {{index: number, program: number, api: string, args: object, response: object}[]} steps - every call in order. * @returns {number} how many responses the model had to act on. */ -function countOpportunities(steps) { +export function countOpportunities(steps) { return steps.filter((step) => { const response = step.response ?? {} if (SIGNAL_STATUSES.includes(response.status)) return true diff --git a/bench/v5-collect.mjs b/bench/v5-collect.mjs new file mode 100644 index 00000000..693e6bd3 --- /dev/null +++ b/bench/v5-collect.mjs @@ -0,0 +1,301 @@ +/** + * The V5 (#994) collector: judge the archived real sessions of one or more + * v5-run sweeps and print the complete denominator with its noise read. + * + * The judge reuses the V4 shared rules verbatim (scanReissues for no-progress + * retries and waits, countOpportunities for correction opportunities) and the + * task verifiers that generalize to real trajectories (the paged-cursor and + * wait-dispatch rules). The scripted-only parts of the V4 stories — the catalog + * track's mandatory wrong call, the load-order track's mandatory not-loaded + * rejection — are deliberately NOT required of a real run: a model that never + * errs must not be judged as failing the story. What the collector reports + * instead, per task and arm, is what #994 asks for: correct next steps, + * correction opportunities, no-progress retries, waits, rule violations, the + * first-round prompt versus the post-load step, and the usage buckets with + * missing usage listed separately. + * + * Nothing here invents a denominator: a session that timed out stays in the + * completion and pass rates, a session without usage stays a missing-usage row, + * and an A/A sweep whose noise band covers a difference makes the collector say + * "not distinguished" instead of attributing the difference to the tier. + * + * @module dsh-ops/bench/v5-collect + */ +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { loadTrackTasks, TRACK_TASK_IDS } from './lib/track-corpus.mjs' +import { countOpportunities, scanReissues, TASK_VERIFIERS } from './lib/tracks.mjs' + +const BENCH_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url))) + +/** The buckets one step's prompt is made of. */ +const PROMPT_BUCKETS = ['inputTokens', 'cacheReadTokens', 'cacheWriteTokens'] +/** The shared usage buckets, in report order. */ +const USAGE_BUCKETS = ['inputTokens', 'outputTokens', 'cacheReadTokens', 'cacheWriteTokens', 'reasoningTokens'] + +/** The CLI text. */ +const USAGE = [ + 'usage: node bench/v5-collect.mjs --run [--run ...] [--out ]', + '', + 'Judges the archived sessions of v5-run sweeps and prints the V5 report tables.', + 'Multiple --run directories merge; the last one carries the output file when', + '--out is omitted.', + '', + 'Exit codes: 0 collected · 2 usage error', +].join('\n') + +/** Parse the command line. */ +function parseArgs(argv) { + const options = { runs: [], out: null } + for (let index = 0; index < argv.length; index += 1) { + const flag = argv[index] + if (flag === '-h' || flag === '--help') { console.log(USAGE); process.exit(0) } + if (flag === '--run') { options.runs.push(path.resolve(argv[index + 1])); index += 1; continue } + if (flag === '--out') { options.out = path.resolve(argv[index + 1]); index += 1; continue } + console.error('unknown option ' + JSON.stringify(flag) + '\n'); console.error(USAGE); process.exit(2) + } + if (options.runs.length === 0) { console.error(USAGE); process.exit(2) } + return options +} + +/** Read one JSON file or fail loud. */ +function readJson(file) { return JSON.parse(fs.readFileSync(file, 'utf8')) } + +/** + * Load one sweep: its environment, its summary rows and every archived trajectory. + * @param {string} runDir - the sweep directory v5-run wrote. + * @returns {object} {environment, rows: [{row, trajectory}]}. + */ +function loadRun(runDir) { + const summaryFile = path.join(runDir, 'summary.json') + if (!fs.existsSync(summaryFile)) throw new Error(summaryFile + ' is missing; run bench/v5-run.mjs first') + const summary = readJson(summaryFile) + const rows = summary.rows.map(row => { + const trajectory = readJson(path.join(runDir, 'results', row.task + '-r' + row.repeat + '.' + row.arm + '.trajectory.json')) + return { row, trajectory } + }) + return { runDir, environment: summary.environment, rows } +} + +/** Flatten one trajectory into the judge's ordered steps. */ +function stepsOf(trajectory) { + const steps = [] + trajectory.programs.forEach((program, programIndex) => { + for (const call of program.calls ?? []) { + steps.push({ index: steps.length, program: programIndex + 1, api: String(call.api), args: call.args ?? {}, response: call.response ?? {} }) + } + }) + return steps +} + +/** A step's dedupe key: same API, same arguments. */ +const stepKey = step => step.api + ' ' + JSON.stringify(step.args) + +/** + * Count one task's correct next steps — the per-task rule, stated plainly: + * - ptc-paged-lookup: reads that start at offset 1 or continue at the pending + * cursor, plus the answer write; + * - ptc-catalog-recover: completed calls that used the task's advertised SDK + * set (the loader counts when the arm publishes one); + * - ptc-sdk-load-order: a completed load followed by completed uses of the + * loaded API in later programs, plus the answer write; + * - ptc-wait-dispatch: completed reads (batching counts once more per extra + * read in one program) and the edit when it awaited an earlier read's + * revision, plus the answer write. + * @param {string} taskId - the task id. + * @param {object[]} steps - the ordered steps. + * @param {object} corpus - the task's fixed facts. + * @param {object} row - the session's folded metrics (for the written answer). + * @returns {number} the count. + */ +function countCorrectNextSteps(taskId, steps, corpus, row) { + const completed = step => step.response?.status === 'completed' + const wroteAnswer = (row.verify?.checks ?? []).some(check => check.passed && /answer/.test(check.path ?? '')) ? 1 : 0 + if (taskId === 'ptc-paged-lookup') { + let correct = 0 + let cursor = null + for (const step of steps.filter(candidate => candidate.api === 'ops_inspect_local_file' && stepArgsPath(step) === corpus.verify.path)) { + const offset = Number(step.args.offset ?? 1) + if (offset === 1 || (cursor !== null && offset === cursor)) correct += 1 + cursor = step.response?.status === 'partial' ? Number(step.response?.data?.nextOffset) : null + } + return correct + wroteAnswer + } + if (taskId === 'ptc-catalog-recover') { + const advertised = new Set([...corpus.sdk, 'ops_load_sdk']) + return steps.filter(step => advertised.has(step.api) && completed(step)).length + wroteAnswer + } + if (taskId === 'ptc-sdk-load-order') { + const load = steps.find(step => step.api === corpus.verify.loader && completed(step)) + const uses = steps.filter(step => step.api === corpus.verify.api) + const good = uses.filter(step => completed(step) && load !== undefined && step.program > load.program).length + return good + wroteAnswer + } + // ptc-wait-dispatch + let correct = 0 + const perProgram = new Map() + for (const step of steps) { + if (step.api === 'ops_inspect_local_file' && completed(step)) { + correct += 1 + perProgram.set(step.program, (perProgram.get(step.program) ?? 0) + 1) + } + } + for (const count of perProgram.values()) if (count >= 2) correct += 1 + const edits = steps.filter(step => step.api === corpus.verify.editApi && stepArgsPath(step) === corpus.verify.winner) + if (edits.length > 0) { + const revision = edits[0].args?.expectedRevision + const awaited = steps.some(step => step.api === 'ops_inspect_local_file' && stepArgsPath(step) === corpus.verify.winner + && completed(step) && step.response?.data?.revision === revision && step.index < edits[0].index) + if (awaited && edits[0].response?.status === 'completed') correct += 2 + } + return correct + wroteAnswer +} + +/** One step's path argument, whatever key it used. */ +function stepArgsPath(step) { return step.args?.path ?? null } + +/** + * Judge one real session: shared rules plus the task rules that generalize. + * @param {object} entry - {row, trajectory}. + * @param {object} corpus - the task's fixed facts. + * @returns {object} the judged block. + */ +function judgeSession(entry, corpus) { + const { row, trajectory } = entry + const steps = stepsOf(trajectory) + const facts = [] + const add = fact => facts.push(fact) + const waits = scanReissues(steps, add) + const opportunities = countOpportunities(steps) + const general = TASK_VERIFIERS[corpus.id] + // The scripted story verifiers demand a mandatory wrong call (catalog) or a + // mandatory not-loaded rejection (load order); a real run that never erred + // must not fail for missing them, so those two task rules stay unused here. + if (corpus.id === 'ptc-paged-lookup' || corpus.id === 'ptc-wait-dispatch') { + general({ programs: trajectory.programs }, corpus, steps, add) + } + if (String(row.answer ?? '').trim() !== String(corpus.answer.content).trim()) { + add('answer.txt: expected ' + short(corpus.answer.content) + ' but the session wrote ' + short(row.answer ?? '')) + } + if (trajectory.programs.length > corpus.budget.maxPrograms) { + add('the session spent ' + trajectory.programs.length + ' programs, over the fixed budget of ' + corpus.budget.maxPrograms) + } + return { + opportunities, waits, + noProgressRetries: facts.filter(fact => fact.startsWith('no-progress retry')).length, + ruleViolations: facts.length, + violations: facts, + correctNextSteps: countCorrectNextSteps(corpus.id, steps, corpus, row), + answerMatch: String(row.answer ?? '').trim() === String(corpus.answer.content).trim(), + } +} + +/** Clamp one fact for a report line. */ +function short(text) { + const value = String(text ?? '') + return value.length > 60 ? value.slice(0, 57) + '...' : value +} + +/** One number's median over a non-empty list; null otherwise. */ +function medianOf(values) { + const usable = values.filter(value => Number.isFinite(value)) + if (usable.length === 0) return null + const sorted = [...usable].sort((left, right) => left - right) + const middle = Math.floor(sorted.length / 2) + return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2 +} + +/** + * Fold one arm's rows of one task into the report block. + * @param {object[]} entries - the {row, trajectory} pairs of that arm. + * @param {object} corpus - the task facts. + * @returns {object} the arm block. + */ +function armBlock(entries, corpus) { + const judged = entries.map(entry => judgeSession(entry, corpus)) + const rows = entries.map(entry => entry.row) + const completedRows = rows.filter(row => !row.timedOut && row.sessionError === null) + const firstPrompts = completedRows.map(row => row.prompt.firstStep).filter(value => Number.isFinite(value)) + const totals = completedRows.map(row => row.prompt.total).filter(value => Number.isFinite(value)) + const arm = { + planned: rows.length, + completed: completedRows.length, + timedOut: rows.filter(row => row.timedOut).length, + errored: rows.filter(row => row.sessionError !== null).length, + missingUsage: rows.filter(row => !row.usageReported).length, + verifyPass: rows.filter(row => row.verify.passed).length, + opportunities: judged.reduce((sum, value) => sum + value.opportunities, 0), + correctNextSteps: judged.reduce((sum, value) => sum + value.correctNextSteps, 0), + noProgressRetries: judged.reduce((sum, value) => sum + value.noProgressRetries, 0), + waits: judged.reduce((sum, value) => sum + value.waits, 0), + ruleViolations: judged.reduce((sum, value) => sum + value.ruleViolations, 0), + firstPrompt: { median: medianOf(firstPrompts), min: firstPrompts.length ? Math.min(...firstPrompts) : null, max: firstPrompts.length ? Math.max(...firstPrompts) : null }, + promptTotal: { median: medianOf(totals), min: totals.length ? Math.min(...totals) : null, max: totals.length ? Math.max(...totals) : null }, + usage: Object.fromEntries(USAGE_BUCKETS.map(bucket => [bucket, completedRows.reduce((sum, row) => sum + Number(row.usage[bucket] ?? 0), 0)])), + cacheMissing: completedRows.filter(row => row.stepsWithUsage !== row.steps || Number.isNaN(Number(row.usage.cacheReadTokens))).length, + judged: judged.map((value, index) => ({ repeat: entries[index].row.repeat, ...value })), + } + return arm +} + +/** Render one block of key/value lines for the report. */ +function lines(label, block, indent = ' ') { + return [indent + '- ' + label + ':'] + .concat(Object.entries(block).filter(([, value]) => typeof value !== 'object').map(([key, value]) => indent + ' ' + key + ': ' + value)) +} + +/** + * The entry point: load every run, judge, and write the report. + * @param {string[]} argv - the arguments after the script name. + */ +export function main(argv = process.argv.slice(2)) { + const options = parseArgs(argv) + const corpusAll = loadTrackTasks() + const runs = options.runs.map(loadRun) + const report = [] + report.push('# V5 Flash comparison — collected tables') + for (const run of runs) { + report.push('') + report.push('## Sweep ' + path.basename(run.runDir) + (run.environment.aa ? ' (A/A)' : '')) + const byTask = new Map() + for (const entry of run.rows) { + if (!byTask.has(entry.row.task)) byTask.set(entry.row.task, new Map()) + const arms = byTask.get(entry.row.task) + arms.set(entry.row.arm, [...(arms.get(entry.row.arm) ?? []), entry]) + } + for (const [taskId, arms] of byTask) { + const corpus = corpusAll.find(entry => entry.id === taskId) + report.push('') + report.push('### Task ' + taskId) + for (const [armLabel, entries] of arms) { + const block = armBlock(entries, corpus) + report.push(...lines('arm ' + armLabel, block)) + for (const bucket of USAGE_BUCKETS) report.push(' usage.' + bucket + ': ' + block.usage[bucket]) + report.push(' firstPrompt: median=' + block.firstPrompt.median + ' min=' + block.firstPrompt.min + ' max=' + block.firstPrompt.max) + report.push(' promptTotal: median=' + block.promptTotal.median + ' min=' + block.promptTotal.min + ' max=' + block.promptTotal.max) + for (const judged of block.judged) { + report.push(' r' + judged.repeat + ': correct=' + judged.correctNextSteps + ' opportunities=' + judged.opportunities + + ' retries=' + judged.noProgressRetries + ' waits=' + judged.waits + ' violations=' + judged.ruleViolations + + ' answer=' + (judged.answerMatch ? 'match' : 'MISMATCH')) + } + } + } + report.push('') + report.push('Environment: plugin ' + run.environment.plugin.commit + ', host ' + run.environment.host.version + + ', ' + run.environment.platform + ', worker sha256 ' + String(run.environment.worker.sha256).slice(0, 8) + + ', preset ' + run.environment.preset + ', condition ' + JSON.stringify(run.environment.condition) + + ', sandbox ' + run.environment.sandbox.mode) + } + const text = report.join('\n') + '\n' + const out = options.out ?? path.join(runs.at(-1).runDir, 'v5-collect.md') + fs.writeFileSync(out, text) + console.log(text) + console.log('[v5] wrote ' + out) + return 0 +} + +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url))) { + process.exitCode = main() +} diff --git a/bench/v5-run.mjs b/bench/v5-run.mjs new file mode 100644 index 00000000..1b192d5a --- /dev/null +++ b/bench/v5-run.mjs @@ -0,0 +1,486 @@ +/** + * The V5 (#994) Flash measurement driver: real OPS-PTC sessions on the real + * opencode-go/deepseek-v4.1-flash route, one fresh composed group per session. + * + * Why this driver exists instead of `bench ab`: the host's one-shot headless + * runner composes the host realm only — it refuses preset sessions by design + * ("runs under agent preset … which the one-shot runner does not compose"), and + * the dsh-ops plugin treats a registry-less host realm as Standard, so an + * `ab` sweep would measure Standard and be invalid per #980. This driver reuses + * the V1 isolated-entry composition (real AgentLoop + real PTC executor + real + * staged worker + the shipped dsh-ops row) and replaces the scripted provider + * with the real llm-pi-ai adapter, which the smoke probe proved end to end: + * run_code-only surface, nested ops_* envelopes, per-step usage with cache + * buckets. + * + * One session = one throwaway workspace + one fresh Context (nothing leaks + * across sessions). The sweep interleaves arms per (task, repeat) AB/BA, so + * machine drift cannot masquerade as an arm effect. Every session archives its + * raw events, its trajectory (root programs + nested calls + envelopes), its + * folded metrics, its verified answer and its workspace outcome; nothing is + * averaged away before it is written down. + * + * Prerequisites (all fail loud, never silently): + * - a built `lib/` and staged `runtime/-/dsh-ops-worker` + * (`npm run build` + `npm run build:ptc`); + * - the vendored `@deepseek-ai/dsh-llm-pi-ai` tree under node_modules (see + * ensureVendoredLlm below — copies the pi-ai adapter and its credential + * helper from the installed dsh host so every import resolves through this + * checkout's tree and one cordis instance serves the whole composition); + * - the route credential: OPENCODE_GO_API_KEY resolved from the credentials + * file of --dsh-home (or already exported in the environment). + * + * @module dsh-ops/bench/v5-run + */ +import { createHash } from 'node:crypto' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { spawnSync } from 'node:child_process' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import { Context } from '@deepseek-ai/cordis' +import Agents from '@deepseek-ai/dsh-agent' +import Sessions from '@deepseek-ai/dsh-session' +import Projections from '@deepseek-ai/dsh-session-projection' +import Prompt from '@deepseek-ai/dsh-system-prompt' +import Tools from '@deepseek-ai/dsh-tools' +import * as PtcExecutor from '@deepseek-ai/dsh-tool-executor-ptc' +import Llm from '@deepseek-ai/dsh-llm' +import * as LlmPiAiModule from '@deepseek-ai/dsh-llm-pi-ai' +import Loop from '@deepseek-ai/dsh-agent-loop' +import { createUserMessage } from '@deepseek-ai/dsh-llm' +import Policy, { setSandboxMode } from '@deepseek-ai/dsh-sandbox-policy' +import Subprocess from '@deepseek-ai/dsh-subprocess-local' +import Sandbox from '@deepseek-ai/dsh-sandbox-local' +import FileSystem from '@deepseek-ai/dsh-fs-local' +import NodePtc from '@deepseek-ai/dsh-ptc-runtime-node' +import { getDshRuntimeVersion } from '@deepseek-ai/dsh-app-boot' + +import { generate } from './lib/fixtures/index.mjs' +import { verifyWorkspace } from './lib/verify.mjs' +import { resolveDshEntry } from './lib/env.mjs' +import { loadTrackTasks, TRACK_TASK_IDS } from './lib/track-corpus.mjs' + +const BENCH_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url))) +const REPO_ROOT = path.dirname(BENCH_DIR) +const LlmPiAi = LlmPiAiModule.default ?? LlmPiAiModule + +/** The frozen route every session of one sweep shares (#994 step 1). */ +const CONDITION = Object.freeze({ provider: 'opencode-go', model: 'deepseek-v4.1-flash', reasoningEffort: 'high' }) +/** The agentPreset every session carries; the plugin's mode detection keys on it. */ +const PRESET = 'ops-ptc' +/** The sandbox mode pinned for every session, matching the operator's real profiles. */ +const SANDBOX_MODE = 'danger-full-access' +/** The two arms of the comparison sweep: the preregistered factor is the row's disclosurePolicy. */ +const ARMS = Object.freeze([ + { label: 'ondemand', disclosurePolicy: 'on-demand' }, + { label: 'full', disclosurePolicy: 'full' }, +]) +/** A/A arms: the same config twice, so the sweep measures the noise floor. */ +const AA_ARMS = Object.freeze([ + { label: 'aa-one', disclosurePolicy: 'on-demand' }, + { label: 'aa-two', disclosurePolicy: 'on-demand' }, +]) + +/** The CLI text. */ +const USAGE = [ + 'usage: node bench/v5-run.mjs [--task ]... [--repeat N] [--aa] [--out ]', + ' [--dsh-home ] [--timeout-ms N] [--keep-workspaces]', + '', + 'Runs real OPS-PTC sessions on the frozen opencode-go/deepseek-v4.1-flash route.', + 'One fresh workspace and one fresh composed group per session; arms interleave', + 'AB/BA per (task, repeat).', + '', + ' --task a V4 track task (' + TRACK_TASK_IDS.join(', ') + ');', + ' default: ptc-catalog-recover ptc-wait-dispatch', + ' --repeat N repeats per task per arm (default 3)', + ' --aa A/A mode: both arms run the on-demand config (default:', + ' the on-demand vs full comparison)', + ' --out the sweep directory (default bench/runs/v5-)', + ' --dsh-home where the credentials file lives (default ~/.dsh)', + ' --timeout-ms N per-session cap (default: the task file\'s own timeoutMs)', + ' --keep-workspaces keep the seeded workspaces under /workspaces', + '', + 'Exit codes: 0 the sweep ran to its plan · 1 a setup or session failure', +].join('\n') + +/** Parse the command line; anything unknown is a usage error. */ +function parseArgs(argv) { + const options = { tasks: [], repeat: 3, aa: false, out: null, dshHome: null, timeoutMs: null, keepWorkspaces: false } + for (let index = 0; index < argv.length; index += 1) { + const flag = argv[index] + const value = argv[index + 1] + if (flag === '-h' || flag === '--help') { console.log(USAGE); process.exit(0) } + if (flag === '--task') { + if (value === undefined || !TRACK_TASK_IDS.includes(value)) { + console.error('--task needs one of: ' + TRACK_TASK_IDS.join(', ')); process.exit(2) + } + options.tasks.push(value); index += 1; continue + } + if (flag === '--repeat') { options.repeat = Number(value); index += 1; continue } + if (flag === '--aa') { options.aa = true; continue } + if (flag === '--out') { options.out = value; index += 1; continue } + if (flag === '--dsh-home') { options.dshHome = value; index += 1; continue } + if (flag === '--timeout-ms') { options.timeoutMs = Number(value); index += 1; continue } + if (flag === '--keep-workspaces') { options.keepWorkspaces = true; continue } + console.error('unknown option ' + JSON.stringify(flag) + '\n'); console.error(USAGE); process.exit(2) + } + if (options.tasks.length === 0) options.tasks = ['ptc-catalog-recover', 'ptc-wait-dispatch'] + if (!Number.isInteger(options.repeat) || options.repeat < 1 || options.repeat > 20) { + console.error('--repeat needs an integer in [1, 20]'); process.exit(2) + } + return options +} + +/** + * Vendor the real pi-ai adapter into this checkout's node_modules so the whole + * composition shares one dsh-llm and one cordis instance. node_modules is + * gitignored and no package.json is touched: this is a runtime resolution + * detail of the measurement driver, not a dependency change. + * @param {string} installed - the installed dsh host's node_modules/@deepseek-ai directory. + */ +function ensureVendoredLlm(installed) { + const root = path.join(REPO_ROOT, 'node_modules') + const targets = [ + { from: path.join(installed, 'dsh-llm-pi-ai'), to: path.join(root, '@deepseek-ai', 'dsh-llm-pi-ai') }, + { from: path.join(installed, 'dsh-credentials'), to: path.join(root, '@deepseek-ai', 'dsh-credentials') }, + ] + for (const target of targets) { + if (fs.existsSync(path.join(target.to, 'package.json'))) continue + if (!fs.existsSync(path.join(target.from, 'package.json'))) { + throw new Error('cannot find the installed ' + path.basename(target.from) + ' under ' + installed) + } + fs.cpSync(target.from, target.to, { recursive: true }) + console.log('[vendor] copied ' + path.basename(target.from) + ' into node_modules (gitignored, no manifest change)') + } + const link = path.join(root, '@earendil-works', 'pi-ai') + const source = path.join(installed, '..', '@earendil-works', 'pi-ai') + if (!fs.existsSync(link)) { + if (!fs.existsSync(path.join(source, 'package.json'))) { + throw new Error('cannot find the installed @earendil-works/pi-ai beside ' + installed) + } + fs.mkdirSync(path.dirname(link), { recursive: true }) + fs.symlinkSync(source, link, 'dir') + console.log('[vendor] linked @earendil-works/pi-ai -> ' + source) + } +} + +/** Resolve OPENCODE_GO_API_KEY from the credentials file or the environment. */ +function resolveCredential(dshHome) { + if (process.env.OPENCODE_GO_API_KEY) return { source: 'environment' } + const file = path.join(dshHome, '.credentials.yaml') + let text = null + try { text = fs.readFileSync(file, 'utf8') } catch { text = null } + const match = text?.match(/^ OPENCODE_GO_API_KEY: (.+)$/m) + if (match === null) throw new Error('OPENCODE_GO_API_KEY is not exported and ' + file + ' does not carry it') + process.env.OPENCODE_GO_API_KEY = match[1].trim() + return { source: file } +} + +/** The checked-out commit, without trusting the environment. */ +function pluginCommit() { + const shaOf = value => /^[0-9a-f]{40}$/i.test(value ?? '') ? value.toLowerCase() : undefined + const result = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: REPO_ROOT, encoding: 'utf8' }) + return shaOf(result.stdout?.trim()) ?? 'unknown' +} + +/** The staged worker the row leaves to its own resolution, with its digest. */ +function stagedWorkerFacts() { + const platform = process.platform + '-' + process.arch + const file = path.join(REPO_ROOT, 'runtime', platform, 'dsh-ops-worker') + if (!fs.existsSync(file)) throw new Error('no staged worker at ' + file + '; run npm run build:ptc') + const receipt = path.join(REPO_ROOT, 'runtime', platform, 'provenance.json') + return { + file, sha256: createHash('sha256').update(fs.readFileSync(file)).digest('hex'), + receipt: fs.existsSync(receipt) ? JSON.parse(fs.readFileSync(receipt, 'utf8')) : null, + } +} + +/** The shipped-row config one arm runs: identical except the preregistered tier. */ +function armRow(arm) { + return { + global: {}, + discovery: { required: true }, + minimal: { enabled: false }, standard: { enabled: false }, cordis: { enabled: false }, ptc: { enabled: false }, + 'ops-ptc': { enabled: true, required: true, disclosurePolicy: arm.disclosurePolicy }, + } +} + +/** + * Boot one session's group: the V1 composition with the real provider. + * @param {string} root - the seeded workspace. + * @param {object} arm - the arm descriptor. + * @returns {Promise<{ctx: object, events: object[], execs: object[], errors: string[]}>} the live group. + */ +async function bootGroup(root, arm) { + const ctx = new Context(); ctx.logger.level = 0 + ctx.provide('workingDirectory', { + get: session => session?.header?.cwd ?? root, + ensure: async agent => agent?.session?.header?.cwd ?? root, + set: async (_agent, target) => target ?? root, + }) + const events = [] + const execs = [] + const errors = [] + for (const plugin of [Agents, Sessions, Projections, Prompt]) await ctx.plugin(plugin, {}) + await ctx.plugin(Tools, {}) + await ctx.plugin(Policy, { mode: SANDBOX_MODE, workspaceRoot: root }) + await ctx.plugin(Subprocess) + await ctx.plugin(Sandbox) + await ctx.plugin(FileSystem, { cwd: root }) + await ctx.plugin(NodePtc, {}) + await ctx.plugin(PtcExecutor, {}) + ctx.on('session/event', (session, event) => events.push(event)) + await ctx.plugin(Llm, {}) + await ctx.plugin(LlmPiAi, { providers: { [CONDITION.provider]: { + apiKeyEnv: 'OPENCODE_GO_API_KEY', + models: [{ id: CONDITION.model, name: CONDITION.model, contextWindow: 1000000, maxTokens: 384000, input: ['text', 'image'] }], + } } }) + ctx.on('tools/execute', async (exec, next) => { const result = await next(); execs.push({ exec, result }); return result }) + ctx.on('agent/error', ({ error }) => errors.push(String(error?.stack ?? error))) + const { apply: applyShippedRow } = await import(pathToFileURL(path.join(REPO_ROOT, 'lib', 'index.js')).href) + await ctx.plugin({ name: 'dsh-ops-v5-row', apply: inner => applyShippedRow(inner, armRow(arm)) }) + await ctx.plugin(Loop, { agents: [], maxParallelToolCalls: 3 }) + await ctx.get('loader')?.await() + return { ctx, events, execs, errors } +} + +/** Parse one exec's arguments whether the host handed a string or an object. */ +function execArgs(exec) { + const raw = exec.input ?? exec.arguments ?? exec.args + if (raw === undefined || raw === null) return {} + if (typeof raw === 'string') { try { return JSON.parse(raw) } catch { return { unparsed: String(raw) } } } + return raw +} + +/** + * Fold a finished session's raw captures into the archived metrics. + * @param {object[]} events - the raw session events, in order. + * @param {object[]} execs - the captured tool executions with results. + * @returns {object} the metrics block. + */ +function foldMetrics(events, execs) { + const usage = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, reasoningTokens: 0 } + const promptSizes = [] + let steps = 0 + let stepsWithUsage = 0 + const toolCallsByName = Object.create(null) + let turnEndReason = null + const number = value => Number.isFinite(Number(value)) ? Number(value) : 0 + for (const event of events) { + const data = event.data ?? {} + if (event.type === 'assistant/message') { + steps += 1 + const reported = data.usage && typeof data.usage === 'object' + if (reported) { + stepsWithUsage += 1 + for (const bucket of Object.keys(usage)) usage[bucket] += number(data.usage[bucket]) + promptSizes.push(number(data.usage.inputTokens) + number(data.usage.cacheReadTokens) + number(data.usage.cacheWriteTokens)) + } + for (const block of data.message?.content ?? []) { + if (block.type === 'tool-call') toolCallsByName[block.name] = (toolCallsByName[block.name] ?? 0) + 1 + } + continue + } + if (event.type === 'turn/end') turnEndReason = data.reason?.kind ?? null + } + return { + steps, stepsWithUsage, + usageReported: steps > 0 && stepsWithUsage === steps, + usage, + prompt: { + reported: steps > 0 && promptSizes.length === steps, + firstStep: promptSizes.length > 0 ? promptSizes[0] : null, + perStep: promptSizes, + total: promptSizes.length > 0 ? promptSizes.reduce((sum, value) => sum + value, 0) : null, + }, + toolCallsByName: { ...toolCallsByName }, + programs: execs.filter(entry => !String(entry.exec.id ?? '').includes(':ptc:')).length, + turnEndReason, + } +} + +/** + * Fold the captured executions into the judge-ready trajectory: root programs + * in completion order, each with its nested calls and envelopes. + * @param {object[]} execs - the captured tool executions with results. + * @returns {object} the trajectory block. + */ +function foldTrajectory(execs) { + const programs = [] + for (const { exec, result } of execs) { + const callId = String(exec.id ?? exec.callId ?? '') + if (callId.includes(':ptc:')) continue + const args = execArgs(exec) + programs.push({ callId, name: exec.name, description: args.description ?? null, code: args.code ?? null, + isError: result?.isError === true, error: result?.error?.message ?? null }) + } + const byRoot = new Map() + for (const { exec, result } of execs) { + const callId = String(exec.id ?? exec.callId ?? '') + const split = callId.lastIndexOf(':ptc:') + if (split < 0) continue + const root = callId.slice(0, split) + if (!byRoot.has(root)) byRoot.set(root, []) + byRoot.get(root).push({ + api: exec.name, + args: execArgs(exec), + response: result?.value ?? { hostError: result?.error?.message ?? 'no envelope' }, + }) + } + for (const program of programs) { + program.calls = byRoot.get(program.callId) ?? [] + } + return { programs } +} + +/** Write one JSON file, refusing to clobber. */ +function writeJson(file, value) { + fs.mkdirSync(path.dirname(file), { recursive: true }) + fs.writeFileSync(file, JSON.stringify(value, null, 2) + '\n') +} + +/** + * Run one measured session end to end and archive it. + * @param {object} plan - {task, repeat, arm, outDir, timeoutMs, corpus}. + * @returns {Promise} the archived row summary. + */ +async function runSession(plan) { + const { task, repeat, arm, outDir, timeoutMs, corpus } = plan + const workspace = fs.mkdtempSync(path.join(outDir, 'workspaces', task.id + '-r' + repeat + '-' + arm.label + '-')) + const fixture = generate(task.task.fixture.generator, task.task.fixture.params ?? {}) + for (const file of fixture.files) { + const target = path.join(workspace, file.path) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, file.content) + } + const group = await bootGroup(workspace, arm) + const { ctx, events, execs, errors } = group + const startedAt = Date.now() + let timedOut = false + let sessionError = null + let agent = null + try { + agent = (await ctx.get('agents').create({ + sessionId: 'v5-' + task.id + '-r' + repeat + '-' + arm.label + '-' + startedAt, + meta: { cwd: workspace, agentPreset: PRESET }, + agentOptions: { ...CONDITION }, + })).agent + setSandboxMode(agent.session, SANDBOX_MODE) + agent.followup(createUserMessage({ content: [{ type: 'text', text: task.task.prompt }], source: { kind: 'user' } })) + await Promise.race([ + agent.whenIdle(), + new Promise((resolve, reject) => { + setTimeout(() => reject(new Error('session timeout')), timeoutMs) + }), + ]) + } catch (error) { + if (/session timeout/.test(String(error?.message))) timedOut = true + else sessionError = String(error?.stack ?? error) + } + const finishedAt = Date.now() + const metrics = foldMetrics(events, execs) + const trajectory = foldTrajectory(execs) + const answerPath = path.join(workspace, 'answer.txt') + const answer = fs.existsSync(answerPath) ? fs.readFileSync(answerPath, 'utf8') : null + const outcome = await verifyWorkspace(workspace, task.task.verify, fixture.exports).catch(error => ({ passed: false, checks: [], error: String(error?.message ?? error) })) + try { await ctx.fiber.dispose() } catch { /* the session is over; dispose is best-effort */ } + if (!plan.keepWorkspaces) fs.rmSync(workspace, { recursive: true, force: true }) + + const row = { + task: task.id, repeat, arm: arm.label, condition: CONDITION, preset: PRESET, + startedAt, finishedAt, durationMs: finishedAt - startedAt, + timedOut, sessionError, agentErrors: errors, + ...metrics, + trajectoryPrograms: trajectory.programs.length, + verify: { passed: outcome.passed === true, checks: (outcome.checks ?? []).map(check => ({ type: check.type, path: check.path, passed: check.passed === true, detail: check.detail ?? '' })) }, + answer, + } + const stem = task.id + '-r' + repeat + '.' + arm.label + writeJson(path.join(outDir, 'results', stem + '.metrics.json'), row) + writeJson(path.join(outDir, 'results', stem + '.trajectory.json'), trajectory) + fs.writeFileSync(path.join(outDir, 'results', stem + '.events.ndjson'), + events.map(event => JSON.stringify(event)).join('\n') + '\n') + return row +} + +/** + * The entry point. + * @param {string[]} argv - the arguments after the script name. + */ +export async function main(argv = process.argv.slice(2)) { + const options = parseArgs(argv) + const dshHome = options.dshHome ?? path.join(os.homedir(), '.dsh') + const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19) + const outDir = options.out ? path.resolve(options.out) : path.join(BENCH_DIR, 'runs', 'v5-' + stamp) + fs.mkdirSync(path.join(outDir, 'results'), { recursive: true }) + fs.mkdirSync(path.join(outDir, 'workspaces'), { recursive: true }) + + // Environment facts, frozen before any session runs (#994 step 1). + const entry = resolveDshEntry(null) + const installedDsh = path.resolve(path.dirname(entry.file), '..') + ensureVendoredLlm(path.join(installedDsh, 'node_modules', '@deepseek-ai')) + const credentialSource = resolveCredential(dshHome) + const worker = stagedWorkerFacts() + const corpus = loadTrackTasks() + const arms = options.aa ? AA_ARMS : ARMS + const environment = { + plugin: { commit: pluginCommit(), root: REPO_ROOT }, + host: { version: getDshRuntimeVersion?.() ?? 'unknown' }, + platform: process.platform + '-' + process.arch, node: process.version, + worker: { file: worker.file, sha256: worker.sha256, receipt: worker.receipt }, + sandbox: { mode: SANDBOX_MODE }, preset: PRESET, condition: CONDITION, + credential: { source: credentialSource.source }, + tasks: options.tasks, repeat: options.repeat, aa: options.aa, + note: 'composed V1 isolated-entry group with the real llm-pi-ai adapter; the host one-shot runner cannot compose presets, so bench ab (host realm = Standard) would be invalid per #980', + } + for (const arm of arms) { + const dump = [ + '# v5 dump-config — arm ' + arm.label + ' (' + new Date().toISOString() + ')', + '# The frozen sweep facts; the row below is the only preregistered difference.', + 'condition: ' + JSON.stringify(CONDITION), + 'preset: ' + PRESET, + 'sandbox: ' + SANDBOX_MODE, + 'row:', + JSON.stringify(armRow(arm), null, 2), + ].join('\n') + fs.mkdirSync(path.join(outDir, 'arms', arm.label), { recursive: true }) + fs.writeFileSync(path.join(outDir, 'arms', arm.label, 'dump-config.txt'), dump + '\n') + } + writeJson(path.join(outDir, 'environment.json'), environment) + + // AB/BA interleave per (task, repeat), as bench/run/schedule.mjs orders a sweep. + const plan = [] + for (const id of options.tasks) { + const task = corpus.find(entry => entry.id === id) + if (task === undefined) { console.error('unknown task ' + id); return 1 } + for (let repeat = 1; repeat <= options.repeat; repeat += 1) { + const order = repeat % 2 === 1 ? [arms[0], arms[1]] : [arms[1], arms[0]] + for (const arm of order) { + plan.push({ task, repeat, arm, outDir, timeoutMs: options.timeoutMs ?? task.task.timeoutMs, keepWorkspaces: options.keepWorkspaces, corpus }) + } + } + } + console.log('[v5] sweep ' + outDir + ' — ' + plan.length + ' sessions: ' + options.tasks.join(', ') + ' x ' + options.repeat + ' x ' + arms.length + ' arms' + (options.aa ? ' (A/A)' : '')) + const rows = [] + for (let index = 0; index < plan.length; index += 1) { + const row = plan[index] + console.log('[v5] ' + (index + 1) + '/' + plan.length + ' — ' + row.task.id + ' r' + row.repeat + ' arm=' + row.arm.label + ' (timeout ' + row.timeoutMs + 'ms)') + const summary = await runSession(row) + rows.push(summary) + console.log('[v5] -> ' + (summary.timedOut ? 'TIMEOUT' : summary.verify.passed ? 'pass' : 'fail') + + ' steps=' + summary.steps + ' usage=' + (summary.usageReported ? 'yes' : 'MISSING') + + ' prompt1=' + summary.prompt.firstStep + ' programs=' + summary.trajectoryPrograms + + ' ' + summary.durationMs + 'ms') + } + writeJson(path.join(outDir, 'summary.json'), { environment, rows }) + const failed = rows.filter(row => row.sessionError !== null).length + console.log('[v5] sweep complete: ' + rows.length + ' sessions, ' + failed + ' with session errors') + console.log('[v5] collect with: node bench/v5-collect.mjs --run ' + outDir) + return failed === 0 ? 0 : 1 +} + +if (process.argv[1] && path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url))) { + process.exitCode = await main() +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index cd455c3c..7b78751a 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,8 @@ this package adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Unreleased] +- Measure the OPS-PTC disclosure tiers on the real Flash route (#994, see `docs/flash-comparison-v5.md`): `bench/v5-run.mjs` runs one fresh composed OPS-PTC group per session on the frozen `opencode-go`/`deepseek-v4.1-flash`/`high` route — the host one-shot runner cannot compose presets (its host realm is Standard, invalid per #980), so the driver reuses the V1 isolated-entry composition with the real `llm-pi-ai` adapter, vendored into `node_modules` at run time — and `bench/v5-collect.mjs` judges the archived trajectories with the V4 shared rules plus the two generalizing task rules, refusing to demand the scripted stories' mandatory wrong call. The pilot (2 tasks x 3 repeats x 2 arms, AB/BA interleaved, plus a 12-session same-condition A/A) reports the complete denominator (24 planned / 24 completed / 0 timed out / 0 missing usage / 23 verified) and the noise verdict: pass rates, retries and opportunities are not distinguished at n=3; the tier's own first-round prompt cost (5.3k vs 30.0k tokens, totals about 2x) is the only difference far outside the A/A band, and no model-quality attribution is made. `bench/lib/tracks.mjs` only gains the two already-shared rule exports. +- 在真实 Flash 路由上测量 OPS-PTC 披露档位(#994,见 `docs/Flash对照-V5.md`):`bench/v5-run.mjs` 每会话在冻结的 `opencode-go`/`deepseek-v4.1-flash`/`high` 路由上组合全新 OPS-PTC 组——宿主一次性 runner 无法组合 preset(host realm 即 Standard,依 #980 无效),故驱动器复用 V1 隔离入口组合并接真实 `llm-pi-ai` 适配器(运行时 vendor 进 `node_modules`);`bench/v5-collect.mjs` 用 V4 共享规则加两条可泛化任务规则判定归档轨迹,且不强制剧本里「必须先犯错」的桥段。试点(2 任务 x 3 重复 x 2 臂,AB/BA 交错,另加 12 会话同条件 A/A)报告完整分母(24 计划 / 24 完成 / 0 超时 / 0 缺 usage / 23 验证通过)与噪声判定:通过率、重试与纠偏机会在 n=3 下未能区分;唯一远超 A/A 带的是档位自身的首轮提示成本(5.3k vs 30.0k tokens,总量约 2x),不作任何模型质量归因。`bench/lib/tracks.mjs` 仅新增两条既有共享规则的导出。 - Turn the two-tier OPS-PTC publication rule into a data-driven acceptance matrix (#992): `test/ptc-two-tier-matrix.test.mjs` runs eight canonical rows (legacy default, the four legacy-off variants, on-demand first step, on-demand after loading, the explicit full first round, read-only, a narrow pool and a custom `profile: 'ptc'` integration) through one shared comparator across three views — the SDK text the model reads, the step admission the loop resolves and the `run_code` program bindings — plus a separately collected no-toolSet diagnostic view that is a registration superset, never an admission proof. Every row executes a real no-side-effect sentinel through the real worker (the read-only row's inspect runs under read-only) and write authority is asserted with a safe rejection naming the refused API. The drift property is itself asserted: removing any single expected name — or reading the diagnostic view where a step view is required — fails at least one cell with the missing/extra names printed, and the unified acceptance holds: the loaded business set equals the full first round on both publication paths. The official `read`/`glob`/`grep` trio is pinned as visible under both `hideOfficialTools` states while #961 is out of baseline, so its landing flips a reviewable pin. No publication behaviour changes. - 把两档 OPS-PTC 发布规则做成数据驱动验收矩阵(#992):`test/ptc-two-tier-matrix.test.mjs` 用同一比较器跑八个规范行(旧缺省、四个旧关闭变体、按需首步、按需加载后、显式全量首轮、只读、窄池、自定义 `profile: 'ptc'` integration)的三个视角——模型读到的 SDK 文本、循环解析的当步准入、`run_code` 程序绑定——外加单列采集的无 toolSet 诊断视图(注册超集,绝非准入证据)。每行都经真实 worker 执行无副作用哨兵(只读行的 inspect 在只读沙箱下真跑),写权限以点名被拒 API 的安全拒绝断言。漂移性质本身被断言:删任一期望名或错读诊断视角,至少一格失败并打印缺失/多余名字;统一口径成立:按需加载完毕的业务集合与全量首轮在两条发布路径上一致。官方 `read`/`glob`/`grep` 三件在 #961 未入基线期间钉为两种 `hideOfficialTools` 状态下均可见,其合入将翻转可评审的钉子。发布行为不变。 - Publish the explicit full PTC disclosure tier (#987): with `disclosurePolicy: 'full'` the `ptc` and `ops-ptc` rows' first round now carries the whole permitted `allowedTools` pool plus `ops_result` (the D1 sign-off item 1 decision, taken conservatively per #979 §3 so refs spilled this step stay consumable — and forced by the unified rule, since on-demand can always load `result`) with no `ops_load_sdk`. The first program can call the permitted pool directly; read-only sessions still exclude every write API, an empty cap stays at resident + `result`, `residentTools:false` drops the resident surface but not the tier, duplicated names register once, `global.dynamicTools:false` keeps its veto (the row publishes its legacy surface letter for letter) and a row that also turns its loader off keeps the tier. `PtcConfig.disclosurePolicy` stays optional because custom integrations keep the legacy on-demand default until #988; the builtin plan carries the parsed value. On-demand (default) behaviour is letter-identical; the settings-page entry is #989. diff --git "a/docs/Flash\345\257\271\347\205\247-V5.md" "b/docs/Flash\345\257\271\347\205\247-V5.md" new file mode 100644 index 00000000..e24de0b7 --- /dev/null +++ "b/docs/Flash\345\257\271\347\205\247-V5.md" @@ -0,0 +1,154 @@ +# Flash 小规模对照(V5) + +[English](flash-comparison-v5.md) | 中文 + +议题 [#994](https://github.com/T-Auto/dsh-ops/issues/994),父单 [#980](https://github.com/T-Auto/dsh-ops/issues/980)。这是一张测量卡: +仓库仅新增测量驱动、采集器、本对报告与 changelog 条目;不改产品代码语义、不改教学文案、不改发布逻辑。 + +## 预注册的问题 + +V5 卡要回答:按需披露档的**更短首轮提示**是否换来**更多调用或更多失败**。两臂只差一个预注册因素—— +`dsh-ops` 行的 `disclosurePolicy`(`on-demand` vs `full`,由 #986/#987 实现)——其余全部冻结: + +- 路由:`opencode-go` / `deepseek-v4.1-flash`,`reasoningEffort: high` +- 预设:`ops-ptc`(组合组自证;若实际落到 Standard,按 #980 判无效) +- 沙箱:`danger-full-access`;平台:`linux-x64` +- 任务:`ptc-catalog-recover`(目录搜索,需要可加载的 `ops_grep`)与 `ptc-wait-dispatch` + (多层配置编辑,需要 `ops_read_files`/`ops_edit_file`)——四条 V4 轨道里工作集对档位最敏感的两条 +- 计划:2 任务 × 3 重复 × 2 臂,按 (任务, 重复) AB/BA 交错,共 12 会话;另跑同条件 A/A + (12 会话,按需配置跑两臂)估计噪声底;每会话使用全新播种工作区与全新组合组 + +## 为什么是驱动器而不是 `bench ab` + +宿主的一次性 headless runner 只组合 host realm,并**按设计拒绝 preset 会话**(其收养检查原话: +*"session … runs under agent preset …, which the one-shot runner does not compose"*)。而插件对无注册表的 +host realm 按 Standard 处理,所以 `ab` 实测的是 Standard,按 #980 判无效。`bench/v5-run.mjs` 复用 V1 +隔离入口组合(真实 AgentLoop、真实 PTC executor、真实 staged worker、正式 `dsh-ops` 行),只把脚本化 +provider 换成真实 `llm-pi-ai` 适配器。一个解析细节:运行时把 `@deepseek-ai/dsh-llm-pi-ai`(连同 +`@deepseek-ai/dsh-credentials`,以及 `@earendil-works/pi-ai` 的符号链接)复制进 `node_modules`, +使全部 import 经本检出树解析、整个组合共享同一个 `cordis` 实例;`node_modules` 被 gitignore, +manifest 不变。每会话落盘:原始事件(`*.events.ndjson`)、可判定轨迹(`*.trajectory.json`)、 +折叠指标(`*.metrics.json`)、经验证的答案,以及每臂 `arms//dump-config.txt`。 + +## 环境事实(记录而非声称) + +| 事实 | 值 | +| --- | --- | +| 插件 commit | `card/v5-flash-comparison` 上 `28c0cf7` = `main@58f6e28` + 已过审前置合并(V1 入口与 V4 轨道 `card/v4-bench-tasks`@`ec4d25c`;被测 D2/D3 披露实现 `card/v3-two-tier-matrix`@`c68fe3d`) | +| 宿主版本 | 0.2.2-rc.1 | +| 平台 | linux-x64,node v24.21.0 | +| worker | staged `runtime/linux-x64/dsh-ops-worker`,sha256 `2b5f988d9115…` | +| 预设 / 路由 | `ops-ptc`;`{provider: opencode-go, model: deepseek-v4.1-flash, reasoningEffort: high}` | +| 沙箱 | danger-full-access、approval never(组合组内钉住模式;未挂审批服务,工具失败为返回值) | + +每臂证据:`bench/runs/v5-comparison/arms/{ondemand,full}/dump-config.txt` 与 +`bench/runs/v5-aa/arms/{aa-one,aa-two}/dump-config.txt`——除一行 `disclosurePolicy` 外逐字相同。 + +## 完整分母 + +| sweep | 计划 | 完成 | 超时 | 缺 usage | 验证通过 | +| --- | --- | --- | --- | --- | --- | +| 对照(按需 vs 全量) | 12 | 12 | 0 | 0 | 11 | +| A/A(按需跑两臂) | 12 | 12 | 0 | 0 | 12 | +| **合计** | **24** | **24** | **0** | **0** | **23** | + +一个对照会话未通过工作区验证:按需臂 `ptc-wait-dispatch` r2 写了 `config/development.json`(选错了 +配置层);会话本身正常完成,因此同时留在完成率与通过率分母里。没有会话超时;每会话每步都报了 +usage,下表没有任何下界问题。正式跑前先做了 2 会话冒烟(`bench/runs/v5-smoke`)验证路由与 usage +采集,冒烟不在预注册分母内。 + +## 按任务结果 + +提示规模为每次模型请求的 token(input + cacheRead + cacheWrite);总量为单会话全部步之和,取臂内三会话中位数。 + +### ptc-catalog-recover + +| 指标 | 按需 | 全量 | A/A 带宽(同配置两臂) | +| --- | --- | --- | --- | +| 验证通过 | 3/3 | 3/3 | 3/3 – 3/3 | +| 正确下一步 | 26 | 17 | 23 – 24 | +| 纠偏机会 | 3 | 0 | 0 – 1 | +| 无进展重试 | 5 | 4 | 5 – 7 | +| 等待 | 0 | 0 | 0 | +| 首轮提示(中位) | 5,271 | 29,946 | 5,271 – 5,271 | +| 提示总量(中位) | 86,136 | 185,465 | 66,881 – 88,078 | +| input tokens(3 会话) | 65,035 | 68,051 | 55,814 – 57,644 | +| cacheRead tokens(3 会话) | 182,656 | 430,848 | 144,896 – 198,016 | + +### ptc-wait-dispatch + +| 指标 | 按需 | 全量 | A/A 带宽 | +| --- | --- | --- | --- | +| 验证通过 | 2/3 | 3/3 | 3/3 – 3/3 | +| 正确下一步 | 8 | 8 | 14 – 27 | +| 纠偏机会 | 1 | 2 | 4 – 6 | +| 无进展重试 | 7 | 6 | 7 – 9 | +| 等待 | 0 | 0 | 0 | +| 首轮提示(中位) | 5,306 | 29,981 | 5,306 – 5,306 | +| 提示总量(中位) | 137,419 | 236,416 | 85,813 – 103,238 | +| input tokens(3 会话) | 117,307 | 106,664 | 65,094 – 125,645 | +| cacheRead tokens(3 会话) | 278,784 | 597,248 | 203,776 – 244,736 | + +### 首轮与加载后的差值 + +`#994` 第 3 步要求列首轮与加载后的差值。按需臂每会话在第一个程序里完成一次 `ops_load_sdk`, +下一个请求因新加载的 SDK 声明加历史而增长: + +| 任务 | 首轮(中位) | 加载完成后的首个请求(中位) | +| --- | --- | --- | +| ptc-catalog-recover | 5,271 | 12,820 | +| ptc-wait-dispatch | 5,306 | 15,574 | + +全量臂按设计没有加载步;首轮即携带整个获准池(29,946 / 29,981 tokens)。A/A 两臂复现按需臂数值 +(12,733–12,786 / 15,437–15,568),这同时也是采集路径的一致性检查。 + +### Cache usage + +每步都报了 usage 对象;每步都有 `cacheReadTokens`,`cacheWriteTokens` 全程为 0(provider 用缓存服务了 +重复的前缀)。缺 usage 行:0。尽管 `reasoningEffort: high`,`reasoningTokens` 每步都为 0——按原样记录, +不做重新解读。 + +## 噪声判定 + +- **未能区分(落在 A/A 噪声内):** `ptc-wait-dispatch` 的验证通过差(2/3 vs 3/3,A/A 带 3/3–3/3, + 即 n=3 下的单次失败;该失败是选错配置层的推理失误,与档位无交互);两任务的无进展重试 + (5 vs 4、7 vs 6,带宽 5–7 与 7–9);`ptc-wait-dispatch` 的纠偏机会(1 vs 2,带宽 4–6,两臂都低于 + A/A 离散本身);两任务的裸 input tokens(仅 `ptc-wait-dispatch` 的 A/A 带就跨 65k–126k)。 +- **超出 A/A 带,但属机械差异,不作模型质量归因:** 首轮提示(5.3k vs 30.0k,带 5.3k–5.3k)与提示 + 总量(86k vs 185k、137k vs 236k,带宽 67k–88k 与 86k–103k)——这是披露档自身的声明成本,输入侧 + 事实而非行为差异;cacheRead 随之(两任务皆 2.4×)。目录任务的纠偏机会(3 vs 0,带 0–1)是按需 + loader 每会话恰好邀请一次可恢复的输入错误(首次 `ops_load_sdk` 名字非法)——三次全部恢复并通过, + 恢复率 3/3,与从未获得机会的全量臂不可区分。目录任务的正确下一步计数(26 vs 17,带 23–24)包含 + 仅存在于按需臂的加载步与恢复步,因此如实报告、不作归因。 +- **不声称档位带来成功率提升。** 每臂每任务 n=3,任何行为侧差异都没有超出 A/A 带所能覆盖的范围。 + +## 复跑 + +```console +npm run build && npm run build:ptc +# 对照 sweep:2 任务 × 3 重复 × 2 臂,AB/BA 交错 +node bench/v5-run.mjs --task ptc-catalog-recover --task ptc-wait-dispatch --repeat 3 \ + --out bench/runs/v5-comparison --dsh-home <.credentials.yaml 内含 OPENCODE_GO_API_KEY 的 home> +# A/A 噪声 sweep:同计划,两臂都用按需配置 +node bench/v5-run.mjs --task ptc-catalog-recover --task ptc-wait-dispatch --repeat 3 --aa \ + --out bench/runs/v5-aa --dsh-home <同一 home> +# 判定与表格 +node bench/v5-collect.mjs --run bench/runs/v5-comparison --run bench/runs/v5-aa +``` + +本报告运行的产物在 `bench/runs/v5-comparison` 与 `bench/runs/v5-aa`(`summary.json`、逐会话 `results/*`、 +每臂 `dump-config.txt`),冒烟在 `bench/runs/v5-smoke`。判定器逐字复用 V4 共享规则 +(`bench/lib/tracks.mjs` 的 `scanReissues`、`countOpportunities`)与两条可泛化的任务规则(分页游标、 +等待/派发)。两条剧本专属规则——目录轨道的"必须先犯错"与加载轨道的"必须先撞未加载拒绝"——有意 +不对真实运行生效:从不犯错的模型不应被判为演砸剧本。 + +## 局限 + +- 每臂每任务 n=3:#994 规定的试点规模。A/A 带展示了该 n 下噪声底有多宽;本报告不支持任何成功率结论。 +- 组合组是 V1 隔离入口,比 web profile 的完整 `ops-ptc` 预设精简(无 persona、time-context、Bash 行)。 + 两臂共享精简组合,档位对比内部有效;绝对 token 数描述的是隔离组合,不是 web profile。 +- 单机、单日、单模型:AB/BA 交错缓解漂移,不消除漂移。 +- 判定器的目录"正确下一步"定义把完成的 `ops_load_sdk` 计为按需臂的正确一步;该列跨臂比较含结构性 + 成分(上文已注明)。 +- 驱动器在运行时把 `dsh-llm-pi-ai` vendor 进 `node_modules`,因为仓库 manifest 有意不依赖任何 LLM + 适配器;该拷贝被 gitignore,并在此处记录。 diff --git a/docs/flash-comparison-v5.md b/docs/flash-comparison-v5.md new file mode 100644 index 00000000..101b9838 --- /dev/null +++ b/docs/flash-comparison-v5.md @@ -0,0 +1,172 @@ +# Flash small-scale comparison (V5) + +English | [中文](Flash对照-V5.md) + +Issue [#994](https://github.com/T-Auto/dsh-ops/issues/994), parent [#980](https://github.com/T-Auto/dsh-ops/issues/980). This is a +measurement card: the only repository additions are the measurement driver, the collector, this report pair and the +changelog entry. No product code, teaching copy or release logic changed. + +## The preregistered question + +The V5 card asks whether the **shorter first-round prompt** of the on-demand PTC disclosure tier trades for **more +calls or more failures**. The two arms therefore differ in exactly one preregistered factor — the `dsh-ops` row's +`disclosurePolicy` (`on-demand` vs `full`, implemented by #986/#987) — while the route, the reasoning effort, the +sandbox mode, the preset, the worker and the task fixtures stay frozen: + +- route: `opencode-go` / `deepseek-v4.1-flash`, `reasoningEffort: high` +- preset: `ops-ptc` (the composed group proves it; a Standard fallback would be invalid per #980) +- sandbox: `danger-full-access`; platform: `linux-x64` +- tasks: `ptc-catalog-recover` (catalog search, needs the loadable `ops_grep`) and `ptc-wait-dispatch` + (multi-layer config edit, needs `ops_read_files`/`ops_edit_file`) — the two V4 tracks whose working set is + most sensitive to the tier +- plan: 2 tasks × 3 repeats × 2 arms, AB/BA interleaved per (task, repeat), 12 sessions; plus a same-condition + A/A sweep (12 sessions, on-demand config twice) for the noise floor; every session ran in a fresh seeded + workspace with a fresh composed group + +## Why a driver instead of `bench ab` + +The host's one-shot headless runner composes the host realm only and **refuses preset sessions by design** (its +adoption check: *"session … runs under agent preset …, which the one-shot runner does not compose"*). The plugin +treats a registry-less host realm as Standard, so an `ab` sweep would measure Standard and be invalid per #980. +`bench/v5-run.mjs` reuses the V1 isolated-entry composition (real AgentLoop, real PTC executor, real staged worker, +the shipped `dsh-ops` row) and replaces the scripted provider with the real `llm-pi-ai` adapter. One resolution +detail: `@deepseek-ai/dsh-llm-pi-ai` (and `@deepseek-ai/dsh-credentials`, plus a symlink for `@earendil-works/pi-ai`) +is copied into `node_modules` at run time so every import resolves through this checkout's tree and one `cordis` +instance serves the whole composition; `node_modules` is gitignored and no manifest changes. The driver archives +per session: raw events (`*.events.ndjson`), the judge-ready trajectory (`*.trajectory.json`), folded metrics +(`*.metrics.json`), the verified answer, and per arm `arms//dump-config.txt`. + +## Environment facts (recorded, not claimed) + +| fact | value | +| --- | --- | +| plugin commit | `28c0cf7` on `card/v5-flash-comparison` = `main@58f6e28` + merged approved prerequisites (V1 entry + V4 tracks `card/v4-bench-tasks` @`ec4d25c`, the tested D2/D3 disclosure implementation `card/v3-two-tier-matrix` @`c68fe3d`) | +| host version | 0.2.2-rc.1 | +| platform | linux-x64, node v24.21.0 | +| worker | staged `runtime/linux-x64/dsh-ops-worker`, sha256 `2b5f988d9115…` | +| preset / route | `ops-ptc`; `{provider: opencode-go, model: deepseek-v4.1-flash, reasoningEffort: high}` | +| sandbox | danger-full-access, approval never (the composed group pins the mode; no approval service is mounted, tool failures are returned values) | + +Per-arm evidence: `bench/runs/v5-comparison/arms/{ondemand,full}/dump-config.txt` and +`bench/runs/v5-aa/arms/{aa-one,aa-two}/dump-config.txt` — identical rows except the one +`disclosurePolicy` line. + +## The complete denominator + +| sweep | planned | completed | timed out | missing usage | verify pass | +| --- | --- | --- | --- | --- | --- | +| comparison (on-demand vs full) | 12 | 12 | 0 | 0 | 11 | +| A/A (on-demand twice) | 12 | 12 | 0 | 0 | 12 | +| **total** | **24** | **24** | **0** | **0** | **23** | + +One comparison session failed the workspace verifier: `ptc-wait-dispatch` r2 on the on-demand arm wrote +`config/development.json` (it picked the wrong config layer); the session completed normally, so it stays in +both the completion and the pass denominator. No session timed out; every step of every session reported usage, so +no metric below is a lower bound. A pre-sweep smoke (2 sessions, `bench/runs/v5-smoke`) confirmed routing and +usage collection first and is not part of the preregistered denominator. + +## Per-task results + +Prompt sizes are tokens per model request (input + cacheRead + cacheWrite); totals are the sum over all steps of +one session, medians over the arm's three sessions. + +### ptc-catalog-recover + +| metric | on-demand | full | A/A band (same config twice) | +| --- | --- | --- | --- | +| verify pass | 3/3 | 3/3 | 3/3 – 3/3 | +| correct next steps | 26 | 17 | 23 – 24 | +| correction opportunities | 3 | 0 | 0 – 1 | +| no-progress retries | 5 | 4 | 5 – 7 | +| waits | 0 | 0 | 0 | +| first-round prompt (median) | 5,271 | 29,946 | 5,271 – 5,271 | +| prompt total (median) | 86,136 | 185,465 | 66,881 – 88,078 | +| input tokens (3 sessions) | 65,035 | 68,051 | 55,814 – 57,644 | +| cacheRead tokens (3 sessions) | 182,656 | 430,848 | 144,896 – 198,016 | + +### ptc-wait-dispatch + +| metric | on-demand | full | A/A band | +| --- | --- | --- | --- | +| verify pass | 2/3 | 3/3 | 3/3 – 3/3 | +| correct next steps | 8 | 8 | 14 – 27 | +| correction opportunities | 1 | 2 | 4 – 6 | +| no-progress retries | 7 | 6 | 7 – 9 | +| waits | 0 | 0 | 0 | +| first-round prompt (median) | 5,306 | 29,981 | 5,306 – 5,306 | +| prompt total (median) | 137,419 | 236,416 | 85,813 – 103,238 | +| input tokens (3 sessions) | 117,307 | 106,664 | 65,094 – 125,645 | +| cacheRead tokens (3 sessions) | 278,784 | 597,248 | 203,776 – 244,736 | + +### First round vs after loading + +`#994` step 3 asks for the first-round vs post-load delta. On the on-demand arm every session completed one +`ops_load_sdk` in its first program, and the next request grew by the loaded SDK declarations plus history: + +| task | first round (median) | first request after the completed load (median) | +| --- | --- | --- | +| ptc-catalog-recover | 5,271 | 12,820 | +| ptc-wait-dispatch | 5,306 | 15,574 | + +The full arm has no load step by design; its first round already carries the whole permitted pool (29,946 / +29,981 tokens). The A/A arms reproduce the on-demand numbers (12,733–12,786 / 15,437–15,568), which is also a +consistency check on the capture path. + +### Cache usage + +Every step reported a usage object; `cacheReadTokens` was present on every step and `cacheWriteTokens` was +reported as 0 throughout (the provider served the repeated prompt prefix from cache). Missing-usage rows: 0. +`reasoningTokens` was 0 on every step despite `reasoningEffort: high` — recorded as-is, not reinterpreted. + +## Noise verdict + +- **Not distinguished (within A/A noise):** the verify-pass difference on `ptc-wait-dispatch` (2/3 vs 3/3 with a + 3/3–3/3 A/A band is a single failure at n=3; the failure was a reasoning mistake — the wrong config layer — + not a tier interaction); no-progress retries on both tasks (5 vs 4 and 7 vs 6 against bands of 5–7 and 7–9); + correction opportunities on `ptc-wait-dispatch` (1 vs 2 against a band of 4–6, i.e. both arms sit below the + A/A spread); bare input tokens on both tasks (the A/A band on `ptc-wait-dispatch` alone spans 65k–126k). +- **Outside the A/A band, but mechanical, not a model-quality claim:** the first-round prompt (5.3k vs 30.0k, + band 5.3k–5.3k) and the prompt totals (86k vs 185k and 137k vs 236k against bands of 67k–88k and 86k–103k) — + this is the disclosure tier's own declaration cost, an input-side fact, not a behavior difference; cacheRead + follows it (2.4× on both tasks). The catalog task's correction opportunities (3 vs 0, band 0–1) are the + on-demand loader inviting exactly one recoverable input error per session (an invalid first + `ops_load_sdk` name) — all three sessions recovered and passed, so the recovery rate is 3/3, not + distinguishable from the full arm which never had the opportunity. The catalog correct-next-step count + (26 vs 17, band 23–24) includes the loader step and the recovery steps that only exist on the on-demand arm, + so it is reported, not attributed. +- **No claim of a success-rate improvement from the tier.** At n=3 per arm per task, nothing behavioral separates + the tiers beyond what the A/A band already covers. + +## Reproduction + +```console +npm run build && npm run build:ptc +# comparison sweep: 2 tasks x 3 repeats x 2 arms, AB/BA interleaved +node bench/v5-run.mjs --task ptc-catalog-recover --task ptc-wait-dispatch --repeat 3 \ + --out bench/runs/v5-comparison --dsh-home +# A/A noise sweep: the same plan with the on-demand config in both arms +node bench/v5-run.mjs --task ptc-catalog-recover --task ptc-wait-dispatch --repeat 3 --aa \ + --out bench/runs/v5-aa --dsh-home +# judge and tables +node bench/v5-collect.mjs --run bench/runs/v5-comparison --run bench/runs/v5-aa +``` + +The artifacts of this report's run live under `bench/runs/v5-comparison` and `bench/runs/v5-aa` +(`summary.json`, per-session `results/*`, per-arm `dump-config.txt`), plus the smoke at `bench/runs/v5-smoke`. +The judge reuses the V4 shared rules verbatim (`scanReissues`, `countOpportunities` in `bench/lib/tracks.mjs`) +and the two task verifiers that generalize (the paged-cursor and wait-dispatch rules). The two scripted-only +story rules — the catalog track's mandatory wrong call and the load-order track's mandatory not-loaded rejection — +are deliberately not required of a real run: a model that never errs must not be judged as failing the story. + +## Limitations + +- n=3 per arm per task: the pilot size #994 prescribes. The A/A bands show how wide the floor is at this n; + nothing here powers a success-rate claim. +- The composed group is the V1 isolated entry, which is leaner than the full web-profile `ops-ptc` preset (no + persona, time-context or Bash row). Both arms share the lean composition, so the tier comparison is internally + valid; absolute token counts describe the isolated composition, not the web profile. +- One machine, one day, one model: drift is mitigated by AB/BA interleaving, not eliminated. +- The judge's catalog "correct next steps" definition counts a completed `ops_load_sdk` as a correct step for the + on-demand arm; cross-arm comparisons of that column are partly structural (documented above). +- The driver vendors `dsh-llm-pi-ai` into `node_modules` at run time because the repository manifest deliberately + does not depend on any LLM adapter; the copy is gitignored and recorded here.