Skip to content

Commit d3aed02

Browse files
V48 (impl-only): LLM call-debug ledger and read first-LLM abort
Add monorepo .tmp/llm-call-debug write path and movable hard-stop after Setup Plan prepare_concise_context reason. Read debug runner uses Haiku.
1 parent ca74f70 commit d3aed02

8 files changed

Lines changed: 1149 additions & 12 deletions

File tree

containers/images/pipeliner/README.md

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,28 @@ docker buildx build \
105105

106106
## Local validation (align with Production)
107107

108+
### Deposit acceptance (seller spine)
109+
110+
From monorepo root — Path **A** LocalHost runner, Path **B** Pipeliner Docker:
111+
112+
```bash
113+
# Real LLM required for measured options:
114+
export BITCODE_ASSET_PACK_REAL_INFERENCE=1
115+
# ANTHROPIC_API_KEY / OPENAI_API_KEY / …
116+
117+
pnpm run accept:deposit-synthesis # Path A (default)
118+
pnpm run accept:deposit-synthesis:docker # Path B (needs Docker)
119+
pnpm run accept:deposit-synthesis:all # A then B
120+
```
121+
122+
Acceptance requires `worthy_deposit_candidates` (or recovered options), **full
123+
absolute measurement catalog** per option, and **inspectable patch/fileChanges**.
124+
Reports land under `.tmp/accept-deposit-a|b/` and `.tmp/accept-deposit-summary/`.
125+
126+
Fixture: `fixtures/deposit-manifest.template.json`.
127+
128+
### Host smoke (entry only)
129+
108130
```bash
109131
# After build:
110132
docker run --rm \
Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"schema": "bitcode.pipeline-host.manifest",
3+
"hostMode": "asset_pack_pipeline",
4+
"synthesizeMode": "deposit",
5+
"sourceRevision": {
6+
"repositoryFullName": "sindresorhus/is-plain-obj",
7+
"branch": "main",
8+
"commit": "REPLACE_WITH_CLONE_SHA"
9+
},
10+
"read": {
11+
"id": "deposit-acceptance-read",
12+
"prompt": "Deposit measured AssetPack options (acceptance fixture)."
13+
},
14+
"deposit": {
15+
"id": "deposit-acceptance",
16+
"hasWalletOrAttestationProof": true,
17+
"hasAssetMeasurementEvidence": false
18+
},
19+
"depositSteering": {
20+
"obfuscations": "Redact secrets, API keys, and private credentials. Prefer public API surface only. Do not include private source bodies in summaries.",
21+
"permissibleSources": [],
22+
"impermissibleSources": ["**/.env*", "**/secrets/**", "**/node_modules/**", "**/*credential*"],
23+
"demandContext": ["library-api", "type-safety", "plain-object-check"]
24+
},
25+
"requireAcceptedReadNeed": false
26+
}

package.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@
6363
"verify:ui": "node scripts/verify-ui-ssot.mjs",
6464
"generate:proven": "node scripts/generate-bitcode-proven.mjs",
6565
"promote:canon": "node scripts/promote-bitcode-canon.mjs",
66+
"accept:deposit-synthesis": "node scripts/accept-deposit-synthesis.mjs --path=a",
67+
"accept:deposit-synthesis:docker": "node scripts/accept-deposit-synthesis.mjs --path=b",
68+
"accept:deposit-synthesis:all": "node scripts/accept-deposit-synthesis.mjs --path=a,b",
69+
"debug:read:first-llm": "pnpm --filter @bitcode/pipeline-hosts run qa:read:debug-first-llm",
6670
"check:canonical-inputs": "node scripts/check-bitcode-canonical-inputs.mjs",
6771
"check:canon-posture-drift": "node scripts/check-bitcode-canon-posture-drift.mjs",
6872
"check:spec-family": "node scripts/check-bitcode-spec-family.mjs",

packages/agent-generics/src/diagnostics/instrumentation.ts

Lines changed: 94 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@ import type { Execution } from '@bitcode/execution-generics/Execution';
33
import { log, writePromptIO, writeStepTraceJSON, writeRawLLMIO } from '@bitcode/logger';
44
import { DIAG_ENABLED, DIAG_TRACES, DIAG_FULL_TRACES, DIAG_FULL_PROMPTS, DIAG_TRACE_MAX, DIAG_WRITE_PROMPT_IO, DIAG_WRITE_STEP_TRACES } from './config';
55
import { collectExecutionTrace, summarizeExecutionTrace } from './trace';
6+
import {
7+
writeLlmCallDebug,
8+
shouldHardStopAfterLlmCall,
9+
getLlmCallDebugRunDir,
10+
} from './llm-call-debug';
611

712
function getCtx(execution: Execution, sequence?: string) {
813
const phase = (execution as any).findUp?.('phase', 'current');
@@ -83,6 +88,36 @@ export async function logLLMSubstepStart(
8388
content: { systemPrompt, userPrompt, combinedPrompt, ...ctx },
8489
}).catch(() => {});
8590

91+
// Monorepo .tmp ledger (call-by-call pipeline debug).
92+
try {
93+
const f = writeLlmCallDebug({
94+
kind: 'request',
95+
sequence,
96+
phase: ctx.phase != null ? String(ctx.phase) : undefined,
97+
agentName: ctx.agentName != null ? String(ctx.agentName) : undefined,
98+
step: ctx.step != null ? String(ctx.step) : undefined,
99+
failsafe: ctx.failsafe != null ? String(ctx.failsafe) : undefined,
100+
path: Array.isArray(ctx.path) ? ctx.path.map(String) : undefined,
101+
correlationId: ctx.correlationId != null ? String(ctx.correlationId) : undefined,
102+
provider: (llmConfig as any)?.provider,
103+
model: llmConfig?.model,
104+
systemPrompt,
105+
userPrompt,
106+
combinedPrompt,
107+
});
108+
if (f) {
109+
log('[llm-call-debug] request written', 'info', {
110+
file: f,
111+
dir: getLlmCallDebugRunDir(String(ctx.correlationId || '')),
112+
agent: ctx.agentName,
113+
step: ctx.step,
114+
failsafe: ctx.failsafe,
115+
generation: sequence,
116+
model: llmConfig?.model,
117+
});
118+
}
119+
} catch { /* never break the pipeline for debug I/O */ }
120+
86121
if (!(DIAG_ENABLED || DIAG_FULL_PROMPTS || DIAG_TRACES)) return;
87122
log('[llm substep] start', 'debug', {
88123
...ctx,
@@ -121,6 +156,34 @@ export async function logLLMSubstepSuccess(
121156
content: { content: output?.content, usage: output?.usage, metadata: output?.metadata },
122157
}).catch(() => {});
123158

159+
try {
160+
const f = writeLlmCallDebug({
161+
kind: 'response',
162+
sequence,
163+
phase: ctx.phase != null ? String(ctx.phase) : undefined,
164+
agentName: ctx.agentName != null ? String(ctx.agentName) : undefined,
165+
step: ctx.step != null ? String(ctx.step) : undefined,
166+
failsafe: ctx.failsafe != null ? String(ctx.failsafe) : undefined,
167+
path: Array.isArray(ctx.path) ? ctx.path.map(String) : undefined,
168+
correlationId: ctx.correlationId != null ? String(ctx.correlationId) : undefined,
169+
provider: output?.metadata?.provider,
170+
model: output?.metadata?.model,
171+
content: output?.content,
172+
usage: output?.usage,
173+
});
174+
if (f) {
175+
log('[llm-call-debug] response written', 'info', {
176+
file: f,
177+
agent: ctx.agentName,
178+
step: ctx.step,
179+
failsafe: ctx.failsafe,
180+
generation: sequence,
181+
model: output?.metadata?.model,
182+
outputLen: String(output?.content || '').length,
183+
});
184+
}
185+
} catch { /* ignore */ }
186+
124187
if (DIAG_ENABLED || DIAG_FULL_PROMPTS || DIAG_TRACES) {
125188
log('[llm substep] success', 'debug', {
126189
...ctx,
@@ -281,18 +344,37 @@ export function logStepTrace(stepExec: Execution, stepName: string) {
281344
// Optional debug-stop helpers to keep core code minimal
282345
export function shouldDebugStopAfterFirstReason(substepExec: Execution, sequence: string): boolean {
283346
try {
284-
if (sequence !== 'reason') return false;
285-
const flag = String(process?.env?.BITCODE_DEBUG_STOP_AFTER_FIRST_REASON || '').toLowerCase() === '1';
286-
if (!flag) return false;
287-
const pathArr = (substepExec as any).getPath?.() || [];
288-
const isPlanStep = pathArr.includes('plan');
289-
const inPrepareFailsafe = pathArr.some((p: string) => String(p).includes('prepare_concise_context'));
290-
const isFirstGen = pathArr.includes('gen-0');
291-
const ctx = getCtx(substepExec);
292-
const agentFilter = process?.env?.BITCODE_DEBUG_STOP_AGENT_FILTER;
293-
const agentMatches = agentFilter ? String(ctx.agentName || '').includes(String(agentFilter)) : true;
294-
if (isPlanStep && inPrepareFailsafe && isFirstGen && agentMatches) {
295-
log('[llm substep] debug-stop', 'info', { ...ctx, reason: 'BITCODE_DEBUG_STOP_AFTER_FIRST_REASON' });
347+
const pathArr: string[] = (substepExec as any).getPath?.() || [];
348+
const ctx = getCtx(substepExec, sequence);
349+
const decision = shouldHardStopAfterLlmCall({
350+
sequence,
351+
pathArr,
352+
phase: ctx.phase != null ? String(ctx.phase) : undefined,
353+
agentName: ctx.agentName != null ? String(ctx.agentName) : undefined,
354+
step: ctx.step != null ? String(ctx.step) : undefined,
355+
failsafe: ctx.failsafe != null ? String(ctx.failsafe) : undefined,
356+
});
357+
if (decision.stop) {
358+
try {
359+
writeLlmCallDebug({
360+
kind: 'abort',
361+
sequence,
362+
phase: ctx.phase != null ? String(ctx.phase) : undefined,
363+
agentName: ctx.agentName != null ? String(ctx.agentName) : undefined,
364+
step: ctx.step != null ? String(ctx.step) : undefined,
365+
failsafe: ctx.failsafe != null ? String(ctx.failsafe) : undefined,
366+
path: pathArr.map(String),
367+
correlationId: ctx.correlationId != null ? String(ctx.correlationId) : undefined,
368+
provider: ctx.provider != null ? String(ctx.provider) : undefined,
369+
model: ctx.model != null ? String(ctx.model) : undefined,
370+
extra: { reason: decision.reason },
371+
});
372+
} catch { /* ignore */ }
373+
log('[llm substep] debug-stop', 'info', {
374+
...ctx,
375+
reason: decision.reason || 'BITCODE_DEBUG_STOP_AFTER_FIRST_REASON',
376+
debugDir: getLlmCallDebugRunDir(String(ctx.correlationId || '')),
377+
});
296378
return true;
297379
}
298380
} catch {}

0 commit comments

Comments
 (0)