V48 Gate 3: Formalize AssetPacksSynthesis onto the real execution/agent/prompt primitives

New asset-packs-synthesis-pipeline.ts runs the measurement on the actual Bitcode primitives instead of the hand-rolled bounded-structured-inference loop:

- Execution spine: PipelineExecution('pipeline:asset-packs-synthesis', lineage{family:asset_pack,posture:live,admittedSurface:lens}) -> phase:<lens> -> factoryAgent (AgentExecution) -> measure step -> createFailsafeGenerationSequence (Failsafe[prepare|chunk|stitch] of Thricified[reason|judge|structured_output]) as REAL nested child nodes.

- Prompt built UPWARDS via the registry: layered PromptParts (pipeline identity + source-safety law / phase lens-role / agent measurement-catalog + rules / step candidate-shape) registered on the AgentExecution.prompt, composed by buildHierarchicalPrompt; structured generation input becomes the user prompt.

- Real ExecutionTool (AssetPackInventoryTool) produces the exclusion-filtered source inventory and is tracked under tool: child execution. AgentExecution auto-resolves the env LLM (no manual wiring). Token accounting walks the execution tree for llm:usage.

Additive: typechecks clean; not yet wired into the route (next chunk rewires synthesizeAssetPackCandidates + the deposit route, then deletes bounded-structured-inference.ts).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: geraldarthurdavis

packages/pipelines/asset-pack/src/asset-packs-synthesis-pipeline.ts

@@ -0,0 +1,285 @@
+/**
+ * AssetPacksSynthesis — formal pipeline execution (V48 Gate 3).
+ *
+ * Runs the synthesis measurement on the real Bitcode primitives instead of a
+ * hand-rolled inference loop:
+ *   PipelineExecution → phase → factoryAgent (AgentExecution) → step
+ *     → createFailsafeGenerationSequence  (Failsafe[prepare|chunk|stitch] ∘
+ *        Thricified[reason|judge|structured_output])
+ *
+ * Every LLM call's prompt is built UPWARDS through the execution's prompt
+ * registry: layered PromptParts registered on the AgentExecution.prompt
+ * (pipeline identity + source-safety, phase lens role, agent measurement
+ * catalog + rules, step candidate shape) compose via buildHierarchicalPrompt.
+ * The exclusion-filtered source inventory is produced by a real ExecutionTool.
+ *
+ * Source-safety: the formal LLM substeps store full prompt/response content,
+ * which is withheld universally by the streaming-layer filter
+ * (sourceSafeStreamEvent). assertSourceSafeCandidates here is the local
+ * defense-in-depth check that no admitted candidate leaks raw source.
+ */
+
+import { z } from 'zod';
+import type { Execution } from '@bitcode/execution-generics/Execution';
+import { PipelineExecution } from '@bitcode/pipelines-generics';
+import {
+  factoryAgent,
+  createFailsafeGenerationSequence,
+  ExecutionTool,
+  type AgentStep,
+} from '@bitcode/agent-generics';
+import { resolveDefaultLLMConfig } from '@bitcode/generic-llms';
+import type { PromptPart } from '@bitcode/prompts/parts/PromptPart';
+
+import {
+  applyExclusionsToInventory,
+  measurementCatalogForLens,
+  type AssetPackMeasurementSpec,
+  type AssetPacksSynthesisLens,
+  type AssetPacksSynthesisSourceInventory,
+  type AssetPacksSynthesisSourceSample,
+  type AssetPacksSynthesisSteering,
+} from './asset-packs-synthesis';
+
+const part = (content: string): PromptPart => content as PromptPart;
+
+// ---- Layered system PromptParts (composed by buildHierarchicalPrompt) -------
+
+const PIPELINE_IDENTITY = part(
+  'You are AssetPacksSynthesis, the single Bitcode synthesis and measurement pipeline. ' +
+    'Depositing and Reading are the same operation at the core: measuring source knowledge ' +
+    'into commercially legible AssetPack candidates; the lens carries the variance.',
+);
+
+const PIPELINE_SOURCE_SAFETY = part(
+  'Source-safety law (mandatory): never quote source code, secrets, or file contents in any ' +
+    'title, summary, or rationale. Describe knowledge and capability, never raw text. Honor the ' +
+    'protected-IP exclusions absolutely — never cover, reference, or describe excluded material.',
+);
+
+const LENS_ROLE: Record<AssetPacksSynthesisLens, PromptPart> = {
+  deposit: part(
+    'Lens: deposit. Depositors supply AssetPacks (bounded, source-safe slices of repository ' +
+      'knowledge) into a Depository where future buyers find Need-fitting packs. Synthesize ' +
+      'distinct deposit AssetPack options the depositor can review and admit.',
+  ),
+  read: part(
+    'Lens: read. Readers ask Bitcode to satisfy a reviewed Need by finding and synthesizing ' +
+      'Need-fitting AssetPacks from Depository source. Synthesize distinct Need-fitting ' +
+      'AssetPack candidates the reader can review and buy.',
+  ),
+};
+
+function buildCatalogPart(catalog: AssetPackMeasurementSpec[]): PromptPart {
+  return part(
+    [
+      'measurements is an object with EXACTLY these keys, each an honest 0..1 volume:',
+      ...catalog.map((spec) => `  ${spec.measurementKind}: ${spec.guidance}`),
+      'Justify every measurement in measurementRationale.',
+    ].join('\n'),
+  );
+}
+
+function buildRulesPart(candidateKinds: string[], maxCandidates: number): PromptPart {
+  return part(
+    [
+      `Synthesize 2-${maxCandidates} DISTINCT candidates from the repository inventory.`,
+      '- coveredSourcePaths must be chosen ONLY from the provided inventory paths, exactly as written.',
+      '- Each candidate is a distinct knowledge slice (different kind and coverage), commercially legible to a buyer.',
+      `- candidate kind must be one of: ${candidateKinds.join(', ')}.`,
+    ].join('\n'),
+  );
+}
+
+function buildShapePart(catalog: AssetPackMeasurementSpec[], maxCandidates: number): PromptPart {
+  return part(
+    [
+      'Return ONLY a JSON object with this EXACT top-level shape (no markdown, no prose, no other top-level key):',
+      `{"options":[{"kind":string,"title":string,"summary":string,"coveredSourcePaths":[string],"measurements":{${catalog
+        .map((spec) => `"${spec.measurementKind}":number`)
+        .join(',')}},"measurementRationale":string,"confidence":number}]}`,
+      `The top-level key MUST be "options" — an array of 2-${maxCandidates} candidate objects. Never return a bare array or any other wrapper key.`,
+    ].join('\n'),
+  );
+}
+
+// ---- Formal source-inventory Tool ------------------------------------------
+
+type InventoryToolArgs = {
+  paths: string[];
+  samples: AssetPacksSynthesisSourceSample[];
+  exclusions: string[];
+};
+
+export class AssetPackInventoryTool extends ExecutionTool<
+  (args: InventoryToolArgs) => Promise<AssetPacksSynthesisSourceInventory>
+> {
+  use = async (args: InventoryToolArgs): Promise<AssetPacksSynthesisSourceInventory> => {
+    return applyExclusionsToInventory(
+      { paths: args.paths, samples: args.samples },
+      args.exclusions,
+    );
+  };
+}
+
+// ---- The formal synthesis run ----------------------------------------------
+
+export interface FormalSynthesisRequest {
+  lens: AssetPacksSynthesisLens;
+  repositoryFullName: string;
+  sourceBranch: string | null;
+  sourceCommit: string | null;
+  steering: AssetPacksSynthesisSteering;
+  inventory: AssetPacksSynthesisSourceInventory;
+  candidateKinds: string[];
+  maxCandidates: number;
+}
+
+export interface FormalSynthesisRawOption {
+  kind: string;
+  title: string;
+  summary: string;
+  coveredSourcePaths: string[];
+  measurements: Record<string, number>;
+  measurementRationale: string;
+  confidence: number;
+}
+
+export interface FormalSynthesisOutcome {
+  options: FormalSynthesisRawOption[];
+  provider: string | null;
+  model: string | null;
+  totalTokens: number | null;
+}
+
+function sumLlmTokens(execution: Execution): number | null {
+  let total = 0;
+  let seen = false;
+  const walk = (node: Execution) => {
+    try {
+      const usage = node.get<Record<string, number>>('llm', 'usage');
+      if (usage && typeof usage === 'object') {
+        const prompt =
+          usage.promptTokens ?? usage.prompt_tokens ?? usage.inputTokens ?? usage.input_tokens ?? 0;
+        const completion =
+          usage.completionTokens ??
+          usage.completion_tokens ??
+          usage.outputTokens ??
+          usage.output_tokens ??
+          0;
+        const t =
+          typeof usage.totalTokens === 'number'
+            ? usage.totalTokens
+            : typeof usage.total_tokens === 'number'
+              ? usage.total_tokens
+              : Number(prompt) + Number(completion);
+        if (Number.isFinite(t) && t > 0) {
+          total += t;
+          seen = true;
+        }
+      }
+    } catch {}
+    for (const child of node.children.values()) walk(child);
+  };
+  walk(execution);
+  return seen ? total : null;
+}
+
+export async function synthesizeAssetPackCandidatesFormal(
+  request: FormalSynthesisRequest,
+  candidateSetSchema: z.ZodType<{ options: FormalSynthesisRawOption[] }>,
+  execution: Execution,
+): Promise<FormalSynthesisOutcome> {
+  const catalog = measurementCatalogForLens(request.lens);
+
+  // Execution spine: Pipeline → Phase → Agent → Step → Generation (real nodes).
+  const pipelineExec = new PipelineExecution(
+    'pipeline:asset-packs-synthesis',
+    execution,
+    {
+      pipelineName: 'asset-packs-synthesis',
+      family: 'asset_pack',
+      posture: 'live',
+      admittedSurface: request.lens,
+    } as any,
+  );
+  const phaseExec = pipelineExec.child(`phase:${request.lens}`);
+
+  // A custom orchestration step (register layered prompts + run the formal
+  // Failsafe∘Thricified generation). Typed as a plain Executor; factoryAgent
+  // only invokes it as a function, so it is cast to AgentStep at the call site.
+  const measureStep = async (
+    _input: unknown,
+    agentExec: Execution,
+  ): Promise<{ options: FormalSynthesisRawOption[] }> => {
+    // Formal Tool: produce the exclusion-filtered inventory and track it.
+    try {
+      (agentExec as any).tools?.registerTool?.('source-inventory', new AssetPackInventoryTool());
+    } catch {}
+    let inventory = request.inventory;
+    try {
+      const tool = (agentExec as any).tools?.getTool?.('source-inventory');
+      if (tool) {
+        inventory = await tool.execute({
+          paths: request.inventory.paths,
+          samples: request.inventory.samples,
+          exclusions: request.steering.protectedIpExclusions,
+        });
+      }
+    } catch {}
+
+    // Layered system prompt parts on the AgentExecution.prompt (compose upwards).
+    try {
+      const p = (agentExec as any).prompt;
+      if (p?.setSpecificExecution) {
+        p.setSpecificExecution('pipeline:asset-packs-synthesis:identity', PIPELINE_IDENTITY);
+        p.setSpecificExecution('pipeline:asset-packs-synthesis:source-safety', PIPELINE_SOURCE_SAFETY);
+        p.setSpecificExecution(`phase:${request.lens}:role`, LENS_ROLE[request.lens]);
+        p.setSpecificExecution('agent:measure:catalog', buildCatalogPart(catalog));
+        p.setSpecificExecution(
+          'agent:measure:rules',
+          buildRulesPart(request.candidateKinds, request.maxCandidates),
+        );
+        p.setSpecificExecution('step:candidate:shape', buildShapePart(catalog, request.maxCandidates));
+      }
+    } catch {}
+
+    // The structured context the generation reasons over (becomes the user prompt).
+    const generationInput = {
+      repository: request.repositoryFullName,
+      branch: request.sourceBranch ?? 'unknown',
+      commit: request.sourceCommit ?? 'unknown',
+      steeringInstructions: request.steering.instructions,
+      protectedIpExclusions: request.steering.protectedIpExclusions,
+      demandContext: request.steering.demandContext,
+      inventoryPaths: inventory.paths,
+      excerpts: inventory.samples,
+    };
+
+    const generated: any = await createFailsafeGenerationSequence<
+      typeof generationInput,
+      { options: FormalSynthesisRawOption[] }
+    >({ outputSchema: candidateSetSchema })(generationInput, agentExec);
+
+    const out = generated?.finalOutput ?? generated?.output ?? generated;
+    return (out && Array.isArray(out.options) ? out : { options: [] }) as {
+      options: FormalSynthesisRawOption[];
+    };
+  };
+
+  const agent = factoryAgent<unknown, { options: FormalSynthesisRawOption[] }>({
+    name: 'asset-packs-measure',
+    description: 'Measure repository source into source-safe AssetPack candidates.',
+    steps: [measureStep as unknown as AgentStep<any, any>],
+  });
+
+  const parsed = await agent(null, phaseExec);
+  const { provider, model } = resolveDefaultLLMConfig();
+
+  return {
+    options: Array.isArray((parsed as any)?.options) ? (parsed as any).options : [],
+    provider: provider ?? null,
+    model: model ?? null,
+    totalTokens: sumLlmTokens(pipelineExec),
+  };
+}