V48 Gate 3 (implementation-only): deposit provisions the full checkout via the Host (retire the GitHub-API sample inventory)

The deposit harness run now provisions the FULL repository checkout on the primitive Host and builds the inventory from it — the measurement reads real full source, not 10 API samples.

- uapi/lib/deposit-source-provisioning.ts: resolveDepositPipelineHost() (InlineHost in-process; the Vercel Sandbox host in prod) + provisionDepositSourceInventory() — provisions via the Host, reads every tracked file verbatim (readWorkspaceSources → sources), derives bounded prompt excerpts (samples), disposes the workspace.
- route: replaced buildSourceInventory (git/trees + ≤10×1600 contents API) with host provisioning at harness-run start (the host clones, not the dispatching request); removed the GitHub-API helpers + sample constants. Emits "Provisioning … on the <host> host" / "Checkout ready: N files (full source measured)".
- asset-packs-synthesis.ts: AssetPacksSynthesisSourceFile + inventory.sources (full verbatim content); applyExclusionsToInventory filters sources fail-closed (protected-IP paths never measured).
- deposit-validation-agent.ts: the absolutes measurement reads inventory.sources (full content) so covered files are measured from real content; samples remain the fallback.
- Tests: provisionDepositSourceInventory (injected fake host — full read, bounded samples, dispose); the route test mocks the Host provisioning (full inventory, exclusions filter both paths AND sources); jest moduleNameMapper + testMatch updated for the new lib + tests/lib.

asset-pack 45 suites/227, uapi deposit provisioning+route+client 8/8, tsc clean (asset-pack + uapi). Inline host is correct in dev today; prod rides the Vercel Sandbox host (standing loose end).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: geraldarthurdavis

packages/pipelines/asset-pack/src/agents/validation/deposit-validation-agent.ts

@@ -301,14 +301,19 @@ export default async function runDepositValidationAgent(input: any, execution: a
   // Per pack in parallel so wall-clock ≈ one measurement; each LLM call is bounded
   // by the F25 per-call timeout.
   if (packs.length > 0) {
-    // The available source for static analysis: the inventory SAMPLES (real,
-    // truncated source). Mapped to {path, content}; the static-analysis tool
-    // measures density from these and applies it to each pack's covered file set.
-    const inventorySources = Array.isArray((inventory as any)?.samples)
-      ? (inventory as any).samples
-          .filter((s: any) => s && typeof s.path === 'string' && typeof s.excerpt === 'string')
-          .map((s: any) => ({ path: s.path as string, content: s.excerpt as string }))
-      : [];
+    // The source for static analysis: the FULL checkout content (inventory.sources —
+    // every tracked file, verbatim, provisioned by the Host) so the covered files are
+    // measured from real content; fall back to the bounded samples when only those
+    // are present (e.g. a pre-Host inventory).
+    const inventorySources = Array.isArray((inventory as any)?.sources)
+      ? (inventory as any).sources
+          .filter((s: any) => s && typeof s.path === 'string' && typeof s.content === 'string')
+          .map((s: any) => ({ path: s.path as string, content: s.content as string }))
+      : Array.isArray((inventory as any)?.samples)
+        ? (inventory as any).samples
+            .filter((s: any) => s && typeof s.path === 'string' && typeof s.excerpt === 'string')
+            .map((s: any) => ({ path: s.path as string, content: s.excerpt as string }))
+        : [];
     await Promise.all(
       packs.map(async (pack: any) => {
         try {

packages/pipelines/asset-pack/src/asset-packs-synthesis.ts

@@ -206,9 +206,22 @@ export interface AssetPacksSynthesisSourceSample {
   excerpt: string;
 }
 
+/** A full-content source file from the host checkout (every blob, verbatim). */
+export interface AssetPacksSynthesisSourceFile {
+  path: string;
+  content: string;
+}
+
 export interface AssetPacksSynthesisSourceInventory {
   paths: string[];
   samples: AssetPacksSynthesisSourceSample[];
+  /**
+   * The FULL verbatim source of the host checkout (V48 Gate 3) — every tracked
+   * file's content, provisioned by the primitive Host. Feeds measurement (the
+   * static-analysis tool reads real full content); the bounded `samples` feed
+   * prompts. Optional for back-compat (absent when only samples were available).
+   */
+  sources?: AssetPacksSynthesisSourceFile[];
   totalPathCount: number;
   excludedPathCount: number;
 }
@@ -334,12 +347,22 @@ export function isPathExcluded(path: string, exclusions: string[]): boolean {
 }
 
 export function applyExclusionsToInventory(
-  inventory: { paths: string[]; samples: AssetPacksSynthesisSourceSample[] },
+  inventory: {
+    paths: string[];
+    samples: AssetPacksSynthesisSourceSample[];
+    sources?: AssetPacksSynthesisSourceFile[];
+  },
   exclusions: string[],
 ): AssetPacksSynthesisSourceInventory {
   const keptPaths = inventory.paths.filter((path) => !isPathExcluded(path, exclusions));
   const keptSamples = inventory.samples.filter((sample) => !isPathExcluded(sample.path, exclusions));
+  // Protected-IP exclusions are honored on the FULL source too (fail-closed): an
+  // excluded file is never measured, sampled, or otherwise carried forward.
+  const keptSources = Array.isArray(inventory.sources)
+    ? inventory.sources.filter((file) => !isPathExcluded(file.path, exclusions))
+    : undefined;
   return {
+    ...(keptSources ? { sources: keptSources } : {}),
     paths: keptPaths,
     samples: keptSamples,
     totalPathCount: inventory.paths.length,

uapi/app/api/deposit/synthesize-options/route.ts

@@ -16,6 +16,10 @@ import {
 import { synthesizeAssetPacksPipeline } from '@bitcode/pipeline-asset-pack';
 import { buildRealDepositAssetPackOptionSynthesis } from '@bitcode/pipeline-asset-pack/deposit-option-real-synthesis';
 import { isAssetPackRealInferenceEnabled } from '@bitcode/pipeline-asset-pack/runtime-inference-policy';
+import {
+  provisionDepositSourceInventory,
+  resolveDepositPipelineHost,
+} from '@/lib/deposit-source-provisioning';
 import {
   bitcodeServerTelemetry,
   compactBitcodeServerId,
@@ -24,9 +28,6 @@ import {
 export const runtime = 'nodejs';
 export const maxDuration = 300;
 
-const MAX_INVENTORY_PATHS = 400;
-const MAX_SAMPLE_FILES = 10;
-const MAX_SAMPLE_CHARS = 1600;
 const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
 
 type SynthesizeOptionsBody = {
@@ -69,78 +70,6 @@ function readSignals(value: unknown) {
     }));
 }
 
-const SAMPLE_PRIORITY_PATTERNS = [
-  /^readme/i,
-  /^package\.json$/i,
-  /^pyproject\.toml$/i,
-  /^cargo\.toml$/i,
-  /^go\.mod$/i,
-  /^setup\.(py|cfg)$/i,
-  /^requirements.*\.txt$/i,
-];
-
-function pickSamplePaths(paths: string[]): string[] {
-  const prioritized = paths.filter((path) =>
-    SAMPLE_PRIORITY_PATTERNS.some((pattern) => pattern.test(path.split('/').pop() || '')),
-  );
-  const sourceLike = paths.filter(
-    (path) =>
-      !prioritized.includes(path) &&
-      /\.(ts|tsx|js|jsx|py|rs|go|rb|java|cs|swift|sol|md)$/i.test(path) &&
-      path.split('/').length <= 3,
-  );
-  return [...prioritized, ...sourceLike].slice(0, MAX_SAMPLE_FILES);
-}
-
-async function githubJson(token: string, url: string) {
-  const response = await fetch(url, {
-    headers: {
-      authorization: `Bearer ${token}`,
-      accept: 'application/vnd.github+json',
-      'x-github-api-version': '2022-11-28',
-    },
-    cache: 'no-store',
-  });
-  if (!response.ok) {
-    throw new Error(`GitHub API ${response.status} for ${url.replace(/\?.*$/, '')}`);
-  }
-  return response.json() as Promise<Record<string, unknown>>;
-}
-
-async function buildSourceInventory(input: {
-  token: string;
-  repositoryFullName: string;
-  reference: string;
-}) {
-  const tree = await githubJson(
-    input.token,
-    `https://api.github.com/repos/${input.repositoryFullName}/git/trees/${encodeURIComponent(input.reference)}?recursive=1`,
-  );
-  const entries = Array.isArray(tree.tree) ? (tree.tree as Array<Record<string, unknown>>) : [];
-  const paths = entries
-    .filter((entry) => entry.type === 'blob' && typeof entry.path === 'string')
-    .map((entry) => entry.path as string)
-    .slice(0, MAX_INVENTORY_PATHS);
-
-  const samples: Array<{ path: string; excerpt: string }> = [];
-  for (const path of pickSamplePaths(paths)) {
-    try {
-      const content = await githubJson(
-        input.token,
-        `https://api.github.com/repos/${input.repositoryFullName}/contents/${encodeURI(path)}?ref=${encodeURIComponent(input.reference)}`,
-      );
-      if (typeof content.content === 'string' && content.encoding === 'base64') {
-        const text = Buffer.from(content.content, 'base64').toString('utf8');
-        samples.push({ path, excerpt: text.slice(0, MAX_SAMPLE_CHARS) });
-      }
-    } catch {
-      // A missing or oversized sample never blocks synthesis.
-    }
-  }
-
-  return { paths, samples, truncated: entries.length > MAX_INVENTORY_PATHS };
-}
-
 export async function POST(request: Request) {
   const supabase = await createClient();
   const {
@@ -274,15 +203,24 @@ export async function POST(request: Request) {
       supabaseAdmin,
     );
     const reference = sourceCommit || sourceBranch || 'HEAD';
-    await emitStatus(`Building source inventory from ${repositoryFullName}@${reference}…`);
-    const rawInventory = await buildSourceInventory({
-      token: auth.accessToken,
+    // Provision the FULL repository checkout on the primitive Host (InlineHost
+    // in-process; the Vercel Sandbox host in prod). The static-analysis measurement
+    // reads the real full source (inventory.sources); bounded samples feed the
+    // prompts. This is the host's job — the dispatching request does not clone.
+    const host = resolveDepositPipelineHost();
+    await emitStatus(
+      `Provisioning ${repositoryFullName}@${reference} on the ${host.capabilities.hostKind} host…`,
+    );
+    const provisioned = await provisionDepositSourceInventory({
+      host,
       repositoryFullName,
-      reference,
+      url: `https://github.com/${repositoryFullName}.git`,
+      revision: reference,
+      token: auth.accessToken,
     });
-    const inventory = applyExclusionsToInventory(rawInventory, protectedIpExclusions);
+    const inventory = applyExclusionsToInventory(provisioned, protectedIpExclusions);
     await emitStatus(
-      `Inventory ready: ${inventory.paths.length} paths (${inventory.excludedPathCount} withheld by ${protectedIpExclusions.length} protected-IP exclusions; ${inventory.samples.length} excerpts sampled).`,
+      `Checkout ready: ${inventory.paths.length} files (${inventory.excludedPathCount} withheld by ${protectedIpExclusions.length} protected-IP exclusions; full source measured, ${inventory.samples.length} prompt excerpts).`,
     );
 
     const DEPOSIT_OPTION_KINDS = ['capability-slice', 'implementation-pattern', 'proof-operations-slice'];

uapi/jest.config.cjs

@@ -84,6 +84,7 @@ module.exports = {
     '^@/lib/bitcoin-wallet-client$': '<rootDir>/lib/bitcoin-wallet-client.ts',
     '^@/lib/github-app-url$': '<rootDir>/lib/github-app-url.ts',
     '^@/lib/bitcode-server-telemetry$': '<rootDir>/lib/bitcode-server-telemetry.ts',
+    '^@/lib/deposit-source-provisioning$': '<rootDir>/lib/deposit-source-provisioning.ts',
     '^@/lib/bitcode-qa-telemetry$': '<rootDir>/lib/bitcode-qa-telemetry.ts',
     '^@/lib/bitcode-wallet-local$': '<rootDir>/lib/bitcode-wallet-local.ts',
     '^@/lib/supabase-auth-redirect$': '<rootDir>/lib/supabase-auth-redirect.ts',
@@ -108,6 +109,8 @@ module.exports = {
     // Include API integration tests for AssetPack and Shippable routes
     '<rootDir>/tests/api/**/*.test.ts',
     '<rootDir>/tests/api/**/*.test.tsx',
+    // Library unit tests (deposit source provisioning, …)
+    '<rootDir>/tests/lib/**/*.test.ts',
     '<rootDir>/tests/webhookRoute.test.ts',
     // Include mock system tests
     '<rootDir>/tests/MockOrchestrator.test.ts',

uapi/lib/deposit-source-provisioning.ts

@@ -0,0 +1,102 @@
+/**
+ * Deposit source provisioning (V48 Gate 3).
+ *
+ * The deposit harness run provisions the FULL repository checkout on the primitive
+ * Host (InlineHost in-process here; the Vercel Sandbox host in prod), then builds the
+ * synthesis inventory FROM the checkout — every tracked file's verbatim content for
+ * measurement (`sources`), plus bounded representative excerpts for the prompts
+ * (`samples`). This retires the GitHub-API sample stopgap; the same `{path, content}`
+ * shape works on either Host implementation.
+ *
+ * Host selection: InlineHost is valid only where the runtime has git + a filesystem
+ * (the dev persistent Node server, NOT a serverless function). Prod deposit runs on
+ * the Vercel Sandbox host (the standing host loose end); when wired,
+ * resolveDepositPipelineHost returns it.
+ */
+
+import {
+  InlineHost,
+  readWorkspaceSources,
+  type BitcodePipelineHost,
+  type HostSourceFile,
+} from '@bitcode/pipeline-hosts';
+
+export interface ProvisionedDepositInventory {
+  paths: string[];
+  samples: { path: string; excerpt: string }[];
+  sources: HostSourceFile[];
+  truncated: boolean;
+}
+
+const SAMPLE_PRIORITY_PATTERNS = [
+  /^readme/i,
+  /^package\.json$/i,
+  /^pyproject\.toml$/i,
+  /^cargo\.toml$/i,
+  /^go\.mod$/i,
+  /^setup\.(py|cfg)$/i,
+  /^requirements.*\.txt$/i,
+];
+const MAX_SAMPLE_FILES = 24;
+const MAX_SAMPLE_CHARS = 4000;
+
+/** Bounded prompt excerpts derived from the real checkout (manifests/README + shallow source). */
+function pickSamples(sources: HostSourceFile[]): { path: string; excerpt: string }[] {
+  const byPath = new Map(sources.map((file) => [file.path, file.content]));
+  const allPaths = sources.map((file) => file.path);
+  const prioritized = allPaths.filter((path) =>
+    SAMPLE_PRIORITY_PATTERNS.some((pattern) => pattern.test(path.split('/').pop() || '')),
+  );
+  const sourceLike = allPaths.filter(
+    (path) =>
+      !prioritized.includes(path) &&
+      /\.(ts|tsx|js|jsx|py|rs|go|rb|java|cs|swift|sol|md)$/i.test(path) &&
+      path.split('/').length <= 3,
+  );
+  return [...prioritized, ...sourceLike]
+    .slice(0, MAX_SAMPLE_FILES)
+    .map((path) => ({ path, excerpt: (byPath.get(path) || '').slice(0, MAX_SAMPLE_CHARS) }));
+}
+
+/**
+ * Resolve the deposit pipeline Host. In-process (InlineHost) today; the Vercel
+ * Sandbox host in prod once wired. The deposit run is the harness run, so the host
+ * provisions the source — the dispatching request does not.
+ */
+export function resolveDepositPipelineHost(): BitcodePipelineHost {
+  return new InlineHost();
+}
+
+/**
+ * Provision the full checkout on the Host and build the deposit inventory from it.
+ * Reads every tracked file's verbatim content (`sources`, for measurement), derives
+ * bounded `samples` (for prompts), and disposes the workspace. The host is passed in
+ * so callers/tests choose the implementation.
+ */
+export async function provisionDepositSourceInventory(input: {
+  host: BitcodePipelineHost;
+  repositoryFullName: string;
+  url: string;
+  revision: string;
+  token?: string;
+}): Promise<ProvisionedDepositInventory> {
+  const workspace = await input.host.provisionRepository({
+    repositoryFullName: input.repositoryFullName,
+    url: input.url,
+    revision: input.revision,
+    username: input.token ? 'x-access-token' : undefined,
+    password: input.token,
+  });
+  try {
+    // Every tracked file, verbatim — the full source the static-analysis measurement reads.
+    const sources = await readWorkspaceSources(workspace);
+    return {
+      paths: sources.map((file) => file.path),
+      samples: pickSamples(sources),
+      sources,
+      truncated: false,
+    };
+  } finally {
+    await workspace.dispose();
+  }
+}