feat(framework): getRequestCompileStats gives a preview's dry-compile render stats

changelog.d/request-compile-stats.added.md

@@ -0,0 +1,8 @@
+- `AgentFramework.getRequestCompileStats(request)` gives the render stats of the dry compile that
+  built a request `previewActivation` returned: context-manager's `CompileResult.renderStats`, what
+  that compile selected, by kind, without the injections merged in after it. So a host can show
+  the makeup of an agent's next request without sending one
+  (anima-research/connectome-host#425). It's null for a request that is sent, when the agent's
+  strategy records no stats, and with context-manager before 0.20.1, whose compiles carry none:
+  agent-framework's own range stays `^0.20.0`, so a host that reads the stats moves its
+  context-manager to 0.20.1.

src/agent.ts

@@ -40,6 +40,23 @@ export interface ActivationRequestOptions {
   membraneRetries?: boolean;
 }
 
+/** A compile's render stats, as context-manager's `getRenderStats()` types
+ *  them; context-manager 0.20.1 gives each compile its own, where the
+ *  strategy records them, in `CompileResult.renderStats`. */
+export type RenderStats = NonNullable<ReturnType<ContextManager['getRenderStats']>>;
+
+/** The render stats of the dry compile behind each request a dry run
+ *  assembled (`assembleActivationRequest` with `dryRun`), by that request.
+ *  A request that is sent has none here. Read through the framework's
+ *  `getRequestCompileStats`. */
+const dryCompileStats = new WeakMap<object, RenderStats | undefined>();
+
+/** The render stats of the dry compile that built `request`, or null: see
+ *  `AgentFramework.getRequestCompileStats`. */
+export function dryCompileStatsOf(request: object): RenderStats | null {
+  return dryCompileStats.get(request) ?? null;
+}
+
 export interface StartStreamResult {
   stream: YieldingStream;
   request: NormalizedRequest;
@@ -1426,7 +1443,7 @@ export class Agent {
           system: this.buildSystemPrompt(prospectiveSystemInjections),
         })
       : undefined;
-    let { messages, systemInjections } = await this.compileWithInjections(
+    const compiled = await this.compileWithInjections(
       budget,
       injections,
       immutablePrefixHash || options.dryRun
@@ -1436,6 +1453,10 @@ export class Agent {
           }
         : undefined,
     );
+    let { messages, systemInjections } = compiled;
+    // context-manager 0.20.1's field, read structurally so this compiles
+    // against 0.20.0 too, whose compiles carry none.
+    const { renderStats } = compiled as typeof compiled & { renderStats?: RenderStats };
 
     // Sanitize: strip empty/whitespace text blocks and drop messages left with
     // no content. The Anthropic API rejects empty text blocks with 400
@@ -1506,6 +1527,7 @@ export class Agent {
       ...this.requestSettings(),
       ...(kvUnified ? { cacheMarkers: 'cm-owned' as const } : {}),
     };
+    if (options.dryRun) dryCompileStats.set(request, renderStats);
     return { request, ...(continuePrompt ? { continuePrompt } : {}) };
   }
 

src/framework.ts

@@ -76,7 +76,7 @@ import { asModuleRow, heldHandle, heldWriteId, matchesMessageQuery } from './hel
 import { ProcessQueueImpl } from './queue.js';
 import { DEFAULT_SHUTDOWN_TIMEOUT_MS, validateShutdownTimeout, shutdownErrorMessage, waitForShutdown, ShutdownTimeoutError } from './shutdown.js';
 import { REFUSAL_REACTIONS, REFUSAL_REACTION_FALLBACK, REFUSAL_REACTION_BASELINE } from './refusal-reactions.js';
-import { Agent, assertAgentSwitches } from './agent.js';
+import { Agent, assertAgentSwitches, dryCompileStatsOf, type RenderStats } from './agent.js';
 import { assertBooleanSetting, holdsSettings } from './config-checks.js';
 import { ModuleRegistry, checkedToolResult, ensureStateRegistered, storableToolResult, thrownResult } from './module-registry.js';
 import { McplServerRegistry } from './mcpl/server-registry.js';
@@ -2616,6 +2616,32 @@ export class AgentFramework {
   getRequestToolPresentation(request: object): PresentationSnapshot | null {
     return this.presentationPreviews.get(request) ?? null;
   }
+  /**
+   * The render stats of the dry compile that built `request`, a request
+   * `previewActivation` returned (or `Agent.buildActivationRequest` with
+   * `dryRun`), as context-manager gives them in `CompileResult.renderStats`:
+   * what that compile's select put in the context, by kind. They count the
+   * selection, not each message of the request. The injections
+   * context-manager merges in after its select, such as those
+   * `previewActivation` gathers with `injections: true`, aren't counted;
+   * context-manager's doc of `CompileResult.renderStats` names what else its
+   * compile does after the select. The agent then changes the messages
+   * itself, and the stats come from before that: they count a message it
+   * drops, such as one left holding only blank text, and leave out the turns
+   * it adds, such as `[Continue]`, a silent heartbeat's separator and
+   * request-only turns. A later preview or live compile leaves them as they
+   * were.
+   * They're the object the compile returned, not a copy, so treat them as
+   * read-only: a change to them shows in every later read. Null for a request
+   * no dry run built, a request that is sent included, and when the agent's
+   * strategy records none (of context-manager's own, the autobiographical
+   * strategy and the knowledge strategy built on it do). Null too with
+   * context-manager before 0.20.1, whose compiles carry no render stats: a
+   * host that reads them moves its own context-manager to 0.20.1.
+   */
+  getRequestCompileStats(request: object): RenderStats | null {
+    return dryCompileStatsOf(request);
+  }
 
   private store: JsStore;
   private ownsStore: boolean;
@@ -7862,6 +7888,9 @@ export class AgentFramework {
    *
    * Pass `{ injections: true }` to opt into full-fidelity gathering and accept
    * those side effects (byte-faithful to a real activation's injected context).
+   *
+   * What the dry compile selected, by kind, is `getRequestCompileStats` of the
+   * request this returns.
    */
   async previewActivation(
     agentName: string,

test/request-compile-stats.test.ts

@@ -0,0 +1,210 @@
+/**
+ * A preview's request carries the render stats of the dry compile that built
+ * it (`getRequestCompileStats`), so a host can show what the next request
+ * would hold, by kind, without sending it (connectome-host#425). The stats
+ * read back for a request are that compile's, whatever was compiled since.
+ * Needs context-manager 0.20.1 or later, whose compiles carry render stats,
+ * and is skipped with 0.20.0, which agent-framework's range allows; with
+ * 0.20.0, the last test checks that a preview's stats are null instead.
+ */
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { AutobiographicalStrategy, KnowledgeStrategy, PassthroughStrategy, type CompileResult, type ContextManager } from '@animalabs/context-manager';
+import { AgentFramework, type Module } from '../src/index.js';
+import { MockMembrane } from './helpers/mock-membrane.js';
+
+/** A module whose gatherContext gives one injection, as a preview with
+ *  `injections: true` gathers it. */
+class Injecting implements Module {
+  readonly name = 'injecting';
+  async start() {}
+  async stop() {}
+  getTools() { return []; }
+  async handleToolCall() { return { success: false, isError: true, error: 'no tools' }; }
+  async onProcess() { return {}; }
+  async gatherContext() {
+    return [{ namespace: 'injecting', position: 'beforeUser' as const, content: [{ type: 'text' as const, text: 'gathered context' }] }];
+  }
+}
+
+async function make() {
+  const dir = mkdtempSync(join(tmpdir(), 'request-compile-stats-'));
+  const framework = await AgentFramework.create({
+    storePath: join(dir, 'store'),
+    membrane: new MockMembrane().asMembrane(),
+    agents: [
+      { name: 'agent', model: 'test-model', systemPrompt: 'test', strategy: new AutobiographicalStrategy() },
+      { name: 'plain', model: 'test-model', systemPrompt: 'test', strategy: new PassthroughStrategy() },
+      { name: 'knowing', model: 'test-model', systemPrompt: 'test', strategy: new KnowledgeStrategy() },
+    ],
+    modules: [new Injecting()],
+  });
+  return {
+    framework,
+    async close() { await framework.stop(); rmSync(dir, { recursive: true, force: true }); },
+  };
+}
+
+/** A compile's result, with context-manager 0.20.1's `renderStats` read
+ *  structurally, as `agent.ts` reads it. */
+type Compiled = CompileResult & { renderStats?: unknown };
+
+/** Every result `cm.compile` returns, in order. */
+function recordResults(cm: ContextManager): Compiled[] {
+  const results: Compiled[] = [];
+  const original = cm.compile.bind(cm);
+  (cm as unknown as { compile: (...args: unknown[]) => Promise<CompileResult> }).compile = async (...args: unknown[]) => {
+    const result = await (original as (...a: unknown[]) => Promise<CompileResult>)(...args);
+    results.push(result);
+    return result;
+  };
+  return results;
+}
+
+const say = (cm: ContextManager, participant: string, text: string) => cm.addMessage(participant, [{ type: 'text', text }]);
+
+/** What a select chose, by bucket: the stats without the parts that can
+ *  differ between two compiles of the same selection, such as a plan's
+ *  timings. */
+const chosen = (s: NonNullable<ReturnType<AgentFramework['getRequestCompileStats']>>) =>
+  ({ head: s.head, tail: s.tail, middleRaw: s.middleRaw, summaries: s.summaries, total: s.total });
+
+/** The installed context-manager's version, from its own package.json. */
+function contextManagerVersion(): string {
+  let dir = new URL('.', import.meta.resolve('@animalabs/context-manager'));
+  for (;;) {
+    const file = new URL('package.json', dir);
+    if (existsSync(file)) {
+      const pkg = JSON.parse(readFileSync(file, 'utf8')) as { name?: string; version: string };
+      if (pkg.name === '@animalabs/context-manager') return pkg.version;
+    }
+    const up = new URL('..', dir);
+    if (up.href === dir.href) throw new Error('found no package.json for @animalabs/context-manager');
+    dir = up;
+  }
+}
+
+// 0.20.0 is the one version agent-framework's range allows whose compiles
+// carry no render stats. Every other version runs these tests, so one that
+// stops carrying them fails here rather than being skipped.
+describe('the render stats of the dry compile behind a preview', {
+  skip: contextManagerVersion() === '0.20.0' ? 'needs context-manager 0.20.1, whose compiles carry render stats' : false,
+}, () => {
+  it('are the ones the compile that built each request returned, and a later preview or live compile leaves an earlier request\'s alone', async () => {
+    const x = await make();
+    try {
+      const agent = x.framework.getAgent('agent')!;
+      const cm = agent.getContextManager();
+      const results = recordResults(cm);
+      say(cm, 'user', 'the first thing said');
+      const first = await x.framework.previewActivation('agent');
+      const firstStats = x.framework.getRequestCompileStats(first);
+      assert.ok(firstStats, 'the autobiographical strategy records stats for a dry compile');
+      assert.equal(firstStats, results[0]!.renderStats, "the first request's are its own compile's");
+      const asRead = structuredClone(firstStats);
+
+      say(cm, 'agent', 'an answer');
+      say(cm, 'user', 'and something more');
+      const second = await x.framework.previewActivation('agent');
+      const secondStats = x.framework.getRequestCompileStats(second);
+      assert.equal(secondStats, results[1]!.renderStats, "the second request's are its own compile's");
+      assert.ok(secondStats!.total.messages > asRead.total.messages, 'setup: the second compile selected more than the first');
+
+      // A live compile, as a turn's request makes it, between the preview and the read.
+      say(cm, 'agent', 'a later answer');
+      await agent.buildActivationRequest([]);
+      assert.ok(results[2]!.renderStats, 'setup: the live compile has stats of its own');
+
+      assert.equal(x.framework.getRequestCompileStats(first), firstStats, 'the first request still reads its own compile\'s stats');
+      assert.deepEqual(firstStats, asRead, 'and the later preview and live compile left them as they were');
+
+      // previewActivation's other path, which gathers injections first.
+      const gathering = await x.framework.previewActivation('agent', { injections: true });
+      assert.ok(results[3]!.renderStats, 'setup: the gathering preview\'s compile has stats');
+      assert.equal(x.framework.getRequestCompileStats(gathering), results[3]!.renderStats, "a preview that gathers injections reads back its own compile's");
+    } finally { await x.close(); }
+  });
+
+  it('are null for a request built to be sent, an object no compile built, and an agent whose strategy records none', async () => {
+    const x = await make();
+    try {
+      const agent = x.framework.getAgent('agent')!;
+      say(agent.getContextManager(), 'user', 'something to compile');
+      const results = recordResults(agent.getContextManager());
+      // buildActivationRequest without dryRun runs the assembly a sent
+      // request goes through, with a live compile: its stats exist, and the
+      // accessor keeps none for it.
+      const sent = await agent.buildActivationRequest([]);
+      assert.ok(results[0]!.renderStats, 'setup: the sent compile has stats of its own');
+      assert.equal(x.framework.getRequestCompileStats(sent), null);
+      assert.equal(x.framework.getRequestCompileStats({}), null);
+
+      const plain = x.framework.getAgent('plain')!.getContextManager();
+      say(plain, 'user', 'something to compile');
+      const plainResults = recordResults(plain);
+      const plainPreview = await x.framework.previewActivation('plain');
+      assert.equal(plainResults.length, 1, 'setup: the passthrough preview compiled');
+      assert.equal(plainResults[0]!.renderStats, undefined, 'setup: a passthrough compile carries no stats');
+      assert.equal(x.framework.getRequestCompileStats(plainPreview), null);
+    } finally { await x.close(); }
+  });
+
+  it('count what the select chose: not the injections or turns added after it, but a message the agent drops after it', async () => {
+    const x = await make();
+    try {
+      const cm = x.framework.getAgent('agent')!.getContextManager();
+      say(cm, 'user', 'a question');
+      const before = await x.framework.previewActivation('agent');
+      const beforeStats = x.framework.getRequestCompileStats(before)!;
+
+      // Context-manager merges gathered injections in after its select.
+      const gathering = await x.framework.previewActivation('agent', { injections: true });
+      assert.ok(JSON.stringify(gathering.messages).includes('gathered context'), 'setup: the injection reached the request');
+      assert.deepEqual(chosen(x.framework.getRequestCompileStats(gathering)!), chosen(beforeStats), 'the injection is not counted');
+
+      // The agent drops a message left holding only blank text after the compile.
+      say(cm, 'user', '   ');
+      const blank = await x.framework.previewActivation('agent');
+      assert.equal(blank.messages.length, before.messages.length, 'setup: the agent dropped the blank message');
+      assert.equal(x.framework.getRequestCompileStats(blank)!.total.messages, beforeStats.total.messages + 1, 'the dropped message is counted');
+
+      // A request that would end on the agent's own message gets a [Continue] turn after the compile.
+      say(cm, 'agent', 'an answer');
+      const continued = await x.framework.previewActivation('agent');
+      assert.equal(continued.messages.length, blank.messages.length + 2, 'setup: the answer and a [Continue] turn');
+      assert.equal(x.framework.getRequestCompileStats(continued)!.total.messages, beforeStats.total.messages + 2, 'the answer is counted and [Continue] is not');
+    } finally { await x.close(); }
+  });
+
+  it('come from the knowledge strategy too, which records them as the autobiographical one it extends does', async () => {
+    const x = await make();
+    try {
+      const cm = x.framework.getAgent('knowing')!.getContextManager();
+      say(cm, 'user', 'something to compile');
+      const results = recordResults(cm);
+      const preview = await x.framework.previewActivation('knowing');
+      assert.ok(results[0]!.renderStats, 'setup: the knowledge strategy\'s compile has stats');
+      assert.equal(x.framework.getRequestCompileStats(preview), results[0]!.renderStats);
+    } finally { await x.close(); }
+  });
+});
+
+describe('the render stats behind a preview, with context-manager 0.20.0', {
+  skip: contextManagerVersion() === '0.20.0' ? false : 'runs only with context-manager 0.20.0, which CI does not install',
+}, () => {
+  it('are null, since its compiles carry none', async () => {
+    const x = await make();
+    try {
+      const cm = x.framework.getAgent('agent')!.getContextManager();
+      say(cm, 'user', 'something to compile');
+      const results = recordResults(cm);
+      const preview = await x.framework.previewActivation('agent');
+      assert.equal(results.length, 1, 'setup: the preview compiled');
+      assert.equal(results[0]!.renderStats, undefined, 'setup: the compile carries no stats');
+      assert.equal(x.framework.getRequestCompileStats(preview), null);
+    } finally { await x.close(); }
+  });
+});