V48 Gate 3 (specification-implementation): deposit Validation agent (quality/coverage/smoke → iterate gate)

geraldarthurdavis · claude · geraldarthurdavis · commit e516e5dd5c0d · 2026-06-25T21:14:03.000-07:00
New DepositValidationAgent (factoryAgentWithPTRR + deterministic smoke/sanity checks) validates the synthesized deposit AssetPacks — measurement honesty (0..1), distinctness, source-safety, obfuscation/exclusion compliance, patch-descriptor coherence, coverage — emitting {issues, qualityScore, coverageGaps, recommendation:'complete'|'iterate'}. Issues stored at validation/implementation:issues (the key the canonical ReadyToFinish gate reads); any issue forces iterate, driving the DIV loop. registerValidationAgentsForType + validationPhase branch on mode (deposit → deposit-quality → ReadyToFinish; read → the 3 parallel validators → ReadyToFinish, untouched). Spec note records the checks + the gate integration. asset-pack 42/208 + typecheck green.

Co-Authored-By: Claude Fable 5 &lt;noreply@anthropic.com&gt;
diff --git a/BITCODE_SPEC_V48_NOTES.md b/BITCODE_SPEC_V48_NOTES.md
@@ -272,7 +272,12 @@ The five phases (same agents in both modes, lens-varied — decided 2026-06-25):
   cloned workspace is the delivery/settlement context.
 - **Validation**: quality thresholds, coverage/corrections/fixes, and AssetPack
   smoke/sanity checks; the Discovery/Implementation/Validation loop iterates until
-  complete or maxIterations.
+  complete or maxIterations. The deposit validator (`validation:deposit-quality`)
+  validates the synthesized AssetPacks — measurement honesty (0..1), distinctness,
+  source-safety, obfuscation/exclusion compliance, patch-descriptor coherence, and
+  coverage — emitting `{issues, qualityScore, coverageGaps, recommendation}`. Its
+  issues are stored at `validation/implementation:issues` (the key the canonical
+  ReadyToFinish gate reads); any issue forces `iterate`.
 - **Finish**: upload the synthesized AssetPack(s) to Bitcode for review.
 
 Implementation status vs this model (Gate-3 build-down list): the deposit
diff --git a/packages/pipelines/asset-pack/src/agents/validation/deposit-validation-agent.ts b/packages/pipelines/asset-pack/src/agents/validation/deposit-validation-agent.ts
@@ -0,0 +1,292 @@
+/**
+ * Deposit-mode Validation agent — Validation phase (V48 Gate 2/3).
+ *
+ * The deposit lens of the SynthesizeAssetPacks Validation phase: validate the
+ * synthesized, measured-patch AssetPacks (implementation:options /
+ * implementation:assetPacks) before Finish uploads them for depositor review.
+ * The read lens validates fits artifacts; the deposit lens validates the
+ * QUALITY of the synthesized supply — measurement honesty, distinctness,
+ * source-safety, obfuscation/exclusion compliance, patch coherence, and
+ * repository coverage — and drives the iterate-vs-complete decision.
+ *
+ * Source-safety (hard constraint): validation describes quality, never raw
+ * source. It reasons only over the source-safe AssetPack descriptors (title,
+ * summary, coveredSourcePaths, measurements, confidence, and the source-safe
+ * patch descriptor of fileChanges path+op + patchSummary), the inventory paths,
+ * the obfuscation guidance, and the protected-IP exclusions — never raw code.
+ *
+ * Runs on the formal PTRR machinery (factoryAgentWithPTRR → Plan/Try/Refine/
+ * Retry) so every call renders in the SDIVF telemetry. The agent's qualitative
+ * findings are merged with deterministic AssetPack smoke/sanity checks, and the
+ * resulting issues are stored under validation/implementation:issues — the exact
+ * store the AssetPack ReadyToFinish gate reads to decide finish vs. review.
+ */
+
+import { factoryAgentWithPTRR } from '@bitcode/agent-generics';
+import { Prompt } from '@bitcode/prompts/prompt';
+import type { PromptPart } from '@bitcode/prompts/parts/PromptPart';
+import { z } from 'zod';
+import { DEPOSIT_MEASUREMENT_CATALOG } from '../../asset-packs-synthesis';
+
+const part = (content: string): PromptPart => content as PromptPart;
+
+const DepositValidationInputSchema = z.object({
+  assetPacks: z.any().optional(),
+  inventory: z.any().optional(),
+  obfuscationGuidance: z.any().optional(),
+  protectedIpExclusions: z.any().optional(),
+});
+
+const DepositValidationOutputSchema = z.object({
+  issues: z.array(z.string()),
+  qualityScore: z.number().min(0).max(1),
+  coverageGaps: z.array(z.string()),
+  recommendation: z.enum(['complete', 'iterate']),
+});
+
+export type DepositValidationResult = z.infer<typeof DepositValidationOutputSchema>;
+
+const IDENTITY = part(
+  'You are the SynthesizeAssetPacks Validation agent in DEPOSIT mode. You validate ' +
+    'the QUALITY of the synthesized, measured-patch AssetPacks the depositor will ' +
+    'review — never the raw source. You reason only over the source-safe AssetPack ' +
+    'descriptors, the repository inventory paths, the obfuscation guidance, and the ' +
+    'protected-IP exclusions. You describe quality and never quote, reconstruct, or ' +
+    'expose raw source, code, secrets, or file contents. Your verdict drives the ' +
+    'iterate-vs-complete decision for the deposit supply.',
+);
+
+const REQUIREMENTS = part(
+  [
+    'Validate the synthesized deposit AssetPacks against these checks and report ' +
+      'every concrete problem as a short, source-safe issue string:',
+    '- Quality threshold: each pack carries honest 0..1 measurements (an object whose ' +
+      'values are in [0,1]) over the expected measurement kinds (' +
+      DEPOSIT_MEASUREMENT_CATALOG.map((spec) => spec.measurementKind).join(', ') +
+      ') with a confidence in [0,1] that is reasonable for the evidence; flag inflated, ' +
+      'out-of-range, or unjustified measurements/confidence.',
+    '- Distinctness: the packs are genuinely distinct, complementary knowledge slices, ' +
+      'not duplicative or near-identical; flag overlap or repetition.',
+    '- Source-safety: NO raw source, code, diffs, secrets, or file contents appear in any ' +
+      'title, summary, measurementRationale, or patchSummary; flag any leakage.',
+    '- Obfuscation/exclusion compliance: no coveredSourcePaths and no patch fileChanges ' +
+      'path touches an obfuscated path/concept (from the obfuscation guidance) or a ' +
+      'protected-IP exclusion; flag any violation by path.',
+    '- Patch coherence: each pack has a source-safe patch descriptor with a non-empty ' +
+      'fileChanges list (path + op = create | modify | delete) and a patchSummary; flag ' +
+      'missing or incoherent patch descriptors.',
+    '- Coverage: the packs adequately cover the repository\'s distinct, buyer-legible ' +
+      'knowledge as represented by the inventory; list the notable uncovered areas in ' +
+      'coverageGaps.',
+    'Set qualityScore in [0,1] as your overall honest quality of the synthesized supply. ' +
+      'Set recommendation to "iterate" when issues or material coverage gaps remain, else ' +
+      '"complete". Return ONLY {"issues":[...],"qualityScore":n,"coverageGaps":[...],' +
+      '"recommendation":"complete"|"iterate"}.',
+  ].join('\n'),
+);
+
+const PLAN = part('Plan: enumerate the synthesized AssetPacks and the quality dimensions to check.');
+const TRY = part('Try: run each quality, distinctness, source-safety, compliance, patch, and coverage check.');
+const REFINE = part('Refine: keep only concrete, source-safe issues and an honest qualityScore and recommendation.');
+const RETRY = part('Retry: when evidence is thin, validate the available AssetPack state and name what is missing.');
+
+function createPrompt(): Prompt {
+  const prompt = new Prompt();
+  prompt.set('agent:identity', IDENTITY);
+  prompt.set('agent:requirements', REQUIREMENTS);
+  prompt.set('ptrr:plan', PLAN);
+  prompt.set('ptrr:try', TRY);
+  prompt.set('ptrr:refine', REFINE);
+  prompt.set('ptrr:retry', RETRY);
+  prompt.require('agent:identity');
+  prompt.require('agent:requirements');
+  prompt.requirePattern('ptrr:*');
+  return prompt;
+}
+
+const prompt = createPrompt();
+
+export const DepositValidationAgent = factoryAgentWithPTRR<
+  z.infer<typeof DepositValidationInputSchema>,
+  DepositValidationResult
+>({
+  name: 'DepositValidationAgent',
+  description:
+    'Validates the synthesized deposit AssetPacks for quality, distinctness, source-safety, obfuscation/exclusion compliance, patch coherence, and coverage (deposit lens).',
+  outputSchema: DepositValidationOutputSchema,
+  tools: [],
+  prompt,
+  stepPrompts: {
+    plan: () => prompt,
+    try: () => prompt,
+    refine: () => prompt,
+    retry: () => prompt,
+  },
+  plan: { chunkThreshold: 2000 },
+  try: { chunkThreshold: 4000 },
+  refine: { maxAttempts: 2 },
+  retry: { maxAttempts: 1 },
+});
+
+function findValue(execution: any, namespace: string, key: string): any {
+  const local = execution?.get?.(namespace, key);
+  if (local !== undefined) return local;
+  return execution?.findUp?.(namespace, key);
+}
+
+function isNum01(value: unknown): boolean {
+  return typeof value === 'number' && Number.isFinite(value) && value >= 0 && value <= 1;
+}
+
+function asPathList(value: unknown): string[] {
+  if (!Array.isArray(value)) return [];
+  return value.filter((p): p is string => typeof p === 'string' && p.length > 0);
+}
+
+// A path violates an exclusion/obfuscation entry when it equals it or sits beneath
+// it as a directory prefix. Source-safe: operates on paths only, never contents.
+function pathViolates(path: string, entry: string): boolean {
+  if (!path || !entry) return false;
+  if (path === entry) return true;
+  const dir = entry.endsWith('/') ? entry : `${entry}/`;
+  return path.startsWith(dir);
+}
+
+// Deterministic AssetPack smoke/sanity checks over the source-safe descriptors.
+// These ground the gate so structurally broken or non-compliant synthesis output
+// always surfaces an issue, independent of the model's qualitative pass.
+function smokeCheckAssetPacks(
+  assetPacks: any[],
+  protectedIpExclusions: string[],
+  obfuscatedPaths: string[],
+): string[] {
+  const issues: string[] = [];
+  if (!Array.isArray(assetPacks) || assetPacks.length === 0) {
+    issues.push('No AssetPacks were synthesized to validate.');
+    return issues;
+  }
+  const forbidden = [...protectedIpExclusions, ...obfuscatedPaths];
+  const seenTitles = new Map<string, number>();
+
+  assetPacks.forEach((pack: any, index: number) => {
+    const label = pack?.title ? `"${pack.title}"` : `#${index + 1}`;
+
+    if (!isNum01(pack?.confidence)) {
+      issues.push(`AssetPack ${label} has a missing or out-of-range confidence (expected 0..1).`);
+    }
+
+    const measurements = pack?.measurements;
+    if (!measurements || typeof measurements !== 'object' || Array.isArray(measurements)) {
+      issues.push(`AssetPack ${label} is missing its 0..1 measurements object.`);
+    } else {
+      for (const [key, value] of Object.entries(measurements)) {
+        if (!isNum01(value)) {
+          issues.push(`AssetPack ${label} measurement "${key}" is not an honest 0..1 volume.`);
+        }
+      }
+    }
+
+    const coveredPaths = asPathList(pack?.coveredSourcePaths);
+    if (coveredPaths.length === 0) {
+      issues.push(`AssetPack ${label} declares no coveredSourcePaths.`);
+    }
+
+    const fileChanges = pack?.patch?.fileChanges;
+    if (!Array.isArray(fileChanges) || fileChanges.length === 0) {
+      issues.push(`AssetPack ${label} has no patch descriptor with fileChanges (patch coherence).`);
+    } else if (typeof pack?.patch?.patchSummary !== 'string' || pack.patch.patchSummary.length === 0) {
+      issues.push(`AssetPack ${label} patch descriptor is missing a source-safe patchSummary.`);
+    }
+
+    // Obfuscation/exclusion compliance over covered paths and patch fileChange paths.
+    const patchPaths = Array.isArray(fileChanges)
+      ? fileChanges.map((fc: any) => fc?.path).filter((p: any): p is string => typeof p === 'string')
+      : [];
+    for (const path of [...coveredPaths, ...patchPaths]) {
+      const hit = forbidden.find((entry) => pathViolates(path, entry));
+      if (hit) {
+        issues.push(`AssetPack ${label} touches withheld path "${path}" (violates exclusion/obfuscation "${hit}").`);
+      }
+    }
+
+    if (typeof pack?.title === 'string' && pack.title.length > 0) {
+      const norm = pack.title.trim().toLowerCase();
+      seenTitles.set(norm, (seenTitles.get(norm) ?? 0) + 1);
+    }
+  });
+
+  for (const [title, count] of seenTitles) {
+    if (count > 1) {
+      issues.push(`AssetPacks are not distinct: ${count} packs share the title "${title}".`);
+    }
+  }
+
+  return issues;
+}
+
+function dedupe(values: string[]): string[] {
+  return Array.from(new Set(values.filter((v) => typeof v === 'string' && v.length > 0)));
+}
+
+export default async function runDepositValidationAgent(input: any, execution: any) {
+  const assetPacks =
+    input?.assetPacks ??
+    findValue(execution, 'implementation', 'options') ??
+    findValue(execution, 'implementation', 'assetPacks') ??
+    [];
+  const inventory = input?.inventory ?? findValue(execution, 'deposit', 'inventory');
+  const obfuscationGuidance =
+    input?.obfuscationGuidance ?? findValue(execution, 'setup', 'inputComprehension');
+  const protectedIpExclusions = asPathList(
+    input?.protectedIpExclusions ?? findValue(execution, 'deposit', 'protectedIpExclusions') ?? [],
+  );
+  const obfuscatedPaths = asPathList((obfuscationGuidance as any)?.obfuscatedPaths);
+  const packs = Array.isArray(assetPacks) ? assetPacks : [];
+
+  const agentOutput = await DepositValidationAgent(
+    {
+      ...input,
+      assetPacks: packs,
+      inventory,
+      inventoryPaths: inventory?.paths,
+      obfuscationGuidance,
+      protectedIpExclusions,
+    },
+    execution,
+  );
+
+  // Deterministic smoke/sanity checks ground the model's qualitative pass.
+  const smokeIssues = smokeCheckAssetPacks(packs, protectedIpExclusions, obfuscatedPaths);
+
+  // Default-fallback to a clean "complete" verdict when the agent returns nothing;
+  // the deterministic smoke issues are always merged so the gate stays grounded.
+  const structured =
+    agentOutput && typeof agentOutput === 'object' && Array.isArray((agentOutput as any).issues);
+  const base: DepositValidationResult = structured
+    ? {
+        issues: Array.isArray((agentOutput as any).issues) ? (agentOutput as any).issues : [],
+        qualityScore: isNum01((agentOutput as any).qualityScore) ? (agentOutput as any).qualityScore : 1,
+        coverageGaps: Array.isArray((agentOutput as any).coverageGaps) ? (agentOutput as any).coverageGaps : [],
+        recommendation: (agentOutput as any).recommendation === 'iterate' ? 'iterate' : 'complete',
+      }
+    : { issues: [], qualityScore: 1, coverageGaps: [], recommendation: 'complete' };
+
+  const issues = dedupe([...base.issues, ...smokeIssues]);
+  const result: DepositValidationResult = {
+    issues,
+    qualityScore: base.qualityScore,
+    coverageGaps: base.coverageGaps,
+    // Any concrete issue (qualitative or smoke) forces an iterate verdict.
+    recommendation: issues.length > 0 ? 'iterate' : base.recommendation,
+  };
+
+  try {
+    // Feed the AssetPack ReadyToFinish gate: it reads validation/implementation:issues
+    // (a bare string[]) to decide finish vs. review. Match that store exactly.
+    execution.store('validation/implementation', 'issues', result.issues);
+    // Record the full deposit-quality verdict for telemetry / the /deposit surface.
+    execution.store('validation', 'depositQuality', result);
+  } catch {}
+
+  return { ...(input || {}), ...result };
+}
diff --git a/packages/pipelines/asset-pack/src/phases/index.ts b/packages/pipelines/asset-pack/src/phases/index.ts
@@ -99,10 +99,18 @@ export const validationPhase: PhaseDelegator<ImplementationOutput, ValidationOut
   // Sequential: validators -> ReadyToFinish
   const readyToFinish = createAgentExecutor('validation:asset-pack-ready-to-finish-agent');
 
-  const exec: Executor<any, any> = sequential(
-    parallelValidators as any,
-    readyToFinish     // Final go/no-go before Finish
-  );
+  // Deposit lens: validate the synthesized AssetPacks for quality (the deposit
+  // validator writes validation/implementation:issues), then the same ReadyToFinish
+  // gate. Read lens keeps the canonical parallel validators -> ReadyToFinish.
+  const exec: Executor<any, any> = mode === 'deposit'
+    ? sequential(
+        createAgentExecutor('validation:deposit-quality'),
+        readyToFinish     // Final go/no-go before Finish
+      )
+    : sequential(
+        parallelValidators as any,
+        readyToFinish     // Final go/no-go before Finish
+      );
   return await exec(input, execution);
 }) as unknown as PhaseDelegator<ImplementationOutput, ValidationOutput>;
 
diff --git a/packages/pipelines/asset-pack/src/phases/validation.ts b/packages/pipelines/asset-pack/src/phases/validation.ts
@@ -1,5 +1,8 @@
 import { createPhaseRunner, PhaseConfig } from '@bitcode/pipelines-generics';
-import { registerValidationAgentsForType as registerAgents } from '../agents/validation-agents';
+import {
+  registerValidationAgentsForType as registerAgents,
+  AssetPackValidationReadyToFinishAgent,
+} from '../agents/validation-agents';
 import { AssetPackWrittenAssetType } from '../types/AssetPackWrittenAssetType';
 import type { SynthesizeAssetPacksMode } from '../synthesize-asset-packs';
 
@@ -25,9 +28,21 @@ export const runValidationPhase = createPhaseRunner(validationPhaseConfig);
 export function registerValidationAgentsForType(
   writtenAssetType: string,
   agentRegistry: any,
-  // mode drives conditional registration: deposit validates synthesized AP
-  // patches for deposit review; read validates fits artifacts. Shared for now.
-  _mode?: SynthesizeAssetPacksMode,
+  // mode drives conditional registration: deposit validates the synthesized AP
+  // patches for deposit review; read validates fits artifacts.
+  mode?: SynthesizeAssetPacksMode,
 ): void {
+  if (mode === 'deposit') {
+    // Deposit lens: a single quality validator over the synthesized AssetPacks,
+    // then the canonical ReadyToFinish gate (it reads validation/implementation:issues).
+    agentRegistry.registerAgent('validation:deposit-quality', () =>
+      import('../agents/validation/deposit-validation-agent').then(m => m.default),
+    );
+    agentRegistry.registerAgent(
+      'validation:asset-pack-ready-to-finish-agent',
+      AssetPackValidationReadyToFinishAgent,
+    );
+    return;
+  }
   registerAgents(writtenAssetType || AssetPackWrittenAssetType.ReadSatisfactionAssetPack, agentRegistry);
 }
