V48 (impl-only): Surface real sandbox host/pipeline failures in UI telemetry

geraldarthurdavis · geraldarthurdavis · commit ee433ddc4a57 · 2026-07-14T23:48:56.000-07:00
Do not mislabel host-command failures as Validation zero-options fail-closed.
Always collect evidence/telemetry on failed host runs; throw formatDepositHostFailure
from stderr/evidence; stream command-completed and pipeline telemetry events to
execution status so the operator sees the actual error.
diff --git a/packages/generic-hosts/VercelSandbox/src/vercel-sandbox-host.ts b/packages/generic-hosts/VercelSandbox/src/vercel-sandbox-host.ts
@@ -149,13 +149,27 @@ export class VercelSandboxPipelineHost {
 
           if (command.required !== false && commandResult.exitCode !== 0) {
             outcome = 'failed';
+            // Keep running artifact collection so UI/telemetry can show stderr
+            // and evidence.error — do not leave the operator with a blank fail.
             break;
           }
         }
 
-        if (outcome === 'completed') {
-          evidence = await this.readJsonArtifact(sandbox, plan.artifactPaths.evidence);
-          telemetry = await this.readTextArtifact(sandbox, plan.artifactPaths.telemetry);
+        // Collect artifacts on completed AND failed runs. Previously only
+        // completed runs read evidence/telemetry, so command failures returned
+        // empty artifacts and the product layer mislabeled them as Validation
+        // zero-options fail-closed.
+        if (outcome === 'completed' || outcome === 'failed') {
+          try {
+            evidence = await this.readJsonArtifact(sandbox, plan.artifactPaths.evidence);
+          } catch {
+            evidence = null;
+          }
+          try {
+            telemetry = await this.readTextArtifact(sandbox, plan.artifactPaths.telemetry);
+          } catch {
+            telemetry = null;
+          }
           await this.emit({
             type: 'artifacts-read',
             timestamp: new Date().toISOString(),
diff --git a/uapi/app/api/deposit/synthesize-options/dispatch-deposit-synthesis.ts b/uapi/app/api/deposit/synthesize-options/dispatch-deposit-synthesis.ts
@@ -208,6 +208,34 @@ export async function runDepositOptionSynthesis(
               void emitStatus(
                 `sandbox: created id=${event.sandboxId ?? event.name ?? 'unknown'} image=${event.image ?? 'none'}`,
               );
+            } else if (event.type === 'command-started') {
+              void emitStatus(`sandbox: command-started ${event.label}`);
+            } else if (event.type === 'command-completed') {
+              const ok = event.exitCode === 0;
+              void emitStatus(
+                `sandbox: command-completed ${event.label} exit=${event.exitCode ?? 'null'}` +
+                  (ok ? '' : ' (FAILED)'),
+              );
+            } else if (event.type === 'artifacts-read') {
+              void emitStatus(
+                `sandbox: artifacts-read evidence=${event.evidencePresent ? 'yes' : 'no'} telemetry=${event.telemetryPresent ? 'yes' : 'no'}`,
+              );
+            } else if (event.type === 'telemetry-artifact-event') {
+              // Surface in-box pipeline progress when the runner writes telemetry.jsonl
+              const te = event.telemetryEvent as {
+                type?: string;
+                stage?: string;
+                message?: string;
+                error?: { message?: string };
+              } | null;
+              const kind = te?.type || 'event';
+              const stage = te?.stage ? ` stage=${te.stage}` : '';
+              const msg = te?.error?.message || te?.message;
+              void emitStatus(
+                `pipeline: ${kind}${stage}${msg ? ` — ${String(msg).slice(0, 280)}` : ''}`,
+              );
+            } else if (event.type === 'sandbox-cancelled') {
+              void emitStatus(`sandbox: cancelled ${event.reason || ''}`.trim());
             } else {
               void emitStatus(`sandbox: ${event.type}`);
             }
@@ -225,6 +253,8 @@ export async function runDepositOptionSynthesis(
         if (hostResult.outcome === 'cancelled') {
           throw new ExecutionCancelledError(runId);
         }
+        // Host failures throw from runDepositInBoxHost with the real command/pipeline
+        // error — never fall through to Validation zero-options fail-closed.
         rawOptions = hostResult.options as Parameters<typeof validateDepositSynthesisOptions>[0];
         inventoryPaths = [
           ...new Set((rawOptions || []).flatMap((option: any) => option?.coveredSourcePaths || [])),
@@ -236,6 +266,9 @@ export async function runDepositOptionSynthesis(
           totalPathCount: inventoryPaths.length,
           excludedPathCount: 0,
         };
+        await emitStatus(
+          `Sandbox host completed with ${Array.isArray(rawOptions) ? rawOptions.length : 0} depositOptions; running fail-closed validation…`,
+        );
       } catch (sandboxError) {
         const message =
           sandboxError instanceof Error ? sandboxError.message : String(sandboxError);
@@ -244,8 +277,10 @@ export async function runDepositOptionSynthesis(
           repositoryFullName,
           runId,
           image: sandboxImage,
-          message: message.slice(0, 600),
+          message: message.slice(0, 800),
         });
+        // Always surface the real host/pipeline error in the execution stream (UI).
+        await emitStatus(`sandbox: failed — ${message.slice(0, 900)}`);
         throw sandboxError instanceof Error
           ? sandboxError
           : new Error(message);
diff --git a/uapi/lib/deposit-source-provisioning.ts b/uapi/lib/deposit-source-provisioning.ts
@@ -135,6 +135,92 @@ export interface DepositInBoxHostResult {
   options: unknown[];
   sandboxId: string | null;
   outcome: PipelineHostRunResult["outcome"];
+  /** Operator-safe failure summary when outcome is failed (no secrets). */
+  failureMessage?: string | null;
+}
+
+const HOST_FAILURE_SNIPPET = 800;
+
+/**
+ * Build an operator-visible failure message from a sandbox host run.
+ * Prefer command stderr, then evidence.error / resultReasons — never invent
+ * Validation "zero options" when the host/pipeline never completed.
+ */
+export function formatDepositHostFailure(result: {
+  outcome?: string | null;
+  sandboxId?: string | null;
+  commands?: Array<{
+    label?: string;
+    exitCode?: number | null;
+    stderr?: string;
+    stdout?: string;
+  }>;
+  artifacts?: {
+    evidence?: unknown | null;
+    telemetry?: string | null;
+  } | null;
+}): string {
+  const parts: string[] = [];
+  const failedCmd = [...(result.commands || [])]
+    .reverse()
+    .find((c) => c.exitCode != null && c.exitCode !== 0 && c.exitCode !== 130);
+  if (failedCmd) {
+    parts.push(
+      `host command "${failedCmd.label || 'unknown'}" exited ${failedCmd.exitCode}`,
+    );
+    const errTail = (failedCmd.stderr || failedCmd.stdout || "").trim();
+    if (errTail) {
+      parts.push(errTail.slice(-HOST_FAILURE_SNIPPET));
+    }
+  }
+
+  const evidence = result.artifacts?.evidence as {
+    error?: { message?: string; name?: string } | null;
+    resultReasons?: unknown;
+    resultState?: unknown;
+    depositOptions?: unknown;
+  } | null;
+
+  if (evidence?.error?.message) {
+    parts.push(
+      `pipeline error: ${String(evidence.error.name || "Error")}: ${String(evidence.error.message).slice(0, 400)}`,
+    );
+  }
+  if (Array.isArray(evidence?.resultReasons) && evidence!.resultReasons!.length > 0) {
+    parts.push(
+      `resultReasons: ${evidence!.resultReasons!.slice(0, 4).map(String).join("; ")}`.slice(
+        0,
+        500,
+      ),
+    );
+  }
+  if (evidence?.resultState) {
+    parts.push(`resultState=${String(evidence.resultState)}`);
+  }
+
+  // Last telemetry lines often hold the real Setup/pipeline stack.
+  const telemetry = result.artifacts?.telemetry;
+  if (typeof telemetry === "string" && telemetry.trim()) {
+    const lines = telemetry.trim().split(/\r?\n/).filter(Boolean);
+    const tail = lines.slice(-3);
+    for (const line of tail) {
+      try {
+        const parsed = JSON.parse(line) as { type?: string; message?: string; error?: { message?: string } };
+        const msg =
+          parsed.error?.message ||
+          parsed.message ||
+          (parsed.type ? `telemetry:${parsed.type}` : null);
+        if (msg) parts.push(String(msg).slice(0, 240));
+      } catch {
+        parts.push(line.slice(0, 240));
+      }
+    }
+  }
+
+  if (parts.length === 0) {
+    return `Sandbox deposit host run failed (outcome=${result.outcome || "failed"}, sandboxId=${result.sandboxId || "none"}). No command stderr or evidence was returned — rebuild Pipeliner image if Setup in-box clone is missing.`;
+  }
+  return `Sandbox deposit host failed: ${parts.join(" | ")}`.slice(0, 1800);
 }
 
 export type DepositSandboxGitRevisionStrategy = GitWorkingTreeStrategy;
@@ -295,30 +381,79 @@ export async function runDepositInBoxHost(input: {
     });
     throw error;
   }
+  const evidenceRaw = result?.artifacts?.evidence as {
+    depositOptions?: unknown;
+    error?: { message?: string } | null;
+    resultState?: unknown;
+  } | null;
+  const optionCount = Array.isArray(evidenceRaw?.depositOptions)
+    ? evidenceRaw!.depositOptions!.length
+    : null;
+  const failedCommands = (result?.commands || []).filter(
+    (c) => c.exitCode != null && c.exitCode !== 0 && c.exitCode !== 130,
+  );
+
   bitcodeServerTelemetry("info", "deposit-sandbox-host", "run-complete", {
     ...createSummary,
     outcome: result?.outcome ?? null,
     sandboxId: result?.sandboxId ?? null,
-    optionCount: Array.isArray(
-      (result?.artifacts?.evidence as { depositOptions?: unknown[] } | null)?.depositOptions,
-    )
-      ? (result!.artifacts!.evidence as { depositOptions: unknown[] }).depositOptions.length
+    optionCount,
+    failedCommandLabels: failedCommands.map((c) => c.label).slice(0, 8),
+    failedExitCodes: failedCommands.map((c) => c.exitCode).slice(0, 8),
+    evidenceResultState: evidenceRaw?.resultState ?? null,
+    evidenceError: evidenceRaw?.error?.message
+      ? String(evidenceRaw.error.message).slice(0, 300)
       : null,
+    hasEvidence: evidenceRaw != null,
+    hasTelemetry: Boolean(result?.artifacts?.telemetry),
   });
+
   if (result?.outcome === "cancelled") {
     return {
       options: [],
       sandboxId: result.sandboxId ?? null,
       outcome: "cancelled",
     };
   }
-  const evidence = result?.artifacts?.evidence as {
-    depositOptions?: unknown;
-  } | null;
+
+  if (result?.outcome === "failed") {
+    const failureMessage = formatDepositHostFailure(result);
+    bitcodeServerTelemetry("error", "deposit-sandbox-host", "host-outcome-failed", {
+      ...createSummary,
+      sandboxId: result.sandboxId ?? null,
+      message: failureMessage.slice(0, 800),
+      failedCommandLabels: failedCommands.map((c) => c.label).slice(0, 8),
+    });
+    const err = new Error(failureMessage) as Error & { hostOutcome?: string };
+    err.hostOutcome = "failed";
+    throw err;
+  }
+
   const options =
-    evidence && Array.isArray(evidence.depositOptions)
-      ? evidence.depositOptions
+    evidenceRaw && Array.isArray(evidenceRaw.depositOptions)
+      ? evidenceRaw.depositOptions
       : [];
+
+  // Completed host with no options is a pipeline/product miss — not Validation
+  // path-admission (that path only applies when options were produced).
+  if (options.length === 0) {
+    const detail = formatDepositHostFailure({
+      ...result,
+      outcome: "completed-empty-options",
+    });
+    const message =
+      `Sandbox deposit pipeline completed with zero depositOptions. ${detail}`.slice(
+        0,
+        1800,
+      );
+    bitcodeServerTelemetry("error", "deposit-sandbox-host", "empty-deposit-options", {
+      ...createSummary,
+      sandboxId: result?.sandboxId ?? null,
+      message: message.slice(0, 800),
+    });
+    throw new Error(message);
+  }
+
   return {
     options,
     sandboxId: result.sandboxId ?? null,
diff --git a/uapi/tests/lib/depositSourceProvisioning.test.ts b/uapi/tests/lib/depositSourceProvisioning.test.ts
@@ -4,6 +4,7 @@
 import {
   buildDepositSandboxGitSource,
   createDepositLocalHostCloneForRun,
+  formatDepositHostFailure,
   provisionDepositCheckout,
   provisionDepositSourceInventory,
   resolveDepositPipelineHost,
@@ -248,22 +249,86 @@ describe('runDepositInBoxHost (#25)', () => {
     expect(receivedPlan.createOptions.name).toMatch(/^bitcode-deposit-/);
   });
 
-  it('returns empty options when the evidence has no depositOptions', async () => {
+  it('throws a host/pipeline error (not Validation zero-options) when outcome is failed', async () => {
     const fakeHost = {
       runHostPlan: async () => ({
-        sandboxId: null,
-        artifacts: { evidence: {}, telemetry: null },
+        sandboxId: 'sbx_fail',
+        artifacts: {
+          evidence: {
+            error: { name: 'Error', message: 'Setup clone failed: Host git clone failed' },
+            resultReasons: ['AssetPack pipeline execution did not produce admissible result evidence.'],
+          },
+          telemetry: null,
+        },
+        outcome: 'failed',
+        stopped: true,
+        manifest: {},
+        commands: [
+          {
+            label: 'asset-pack-pipeline-run',
+            exitCode: 1,
+            stderr: 'Error: Setup clone failed: Host git clone failed\n',
+            stdout: '',
+          },
+        ],
+      }),
+    };
+    await expect(
+      runDepositInBoxHost({
+        repositoryFullName: 'o/r',
+        revision: 'main',
+        branch: 'main',
+        commit: null,
+        obfuscations: null,
+        forcedExclusions: [],
+        demandContext: [],
+        hostFactory: async () => fakeHost,
+      }),
+    ).rejects.toThrow(/Sandbox deposit host failed|Setup clone failed|asset-pack-pipeline-run/);
+  });
+
+  it('throws when host completed with zero depositOptions (distinct from Validation)', async () => {
+    const fakeHost = {
+      runHostPlan: async () => ({
+        sandboxId: 'sbx_empty',
+        artifacts: { evidence: { depositOptions: [], resultState: 'blocked_readiness' }, telemetry: null },
         outcome: 'completed',
         stopped: true,
         manifest: {},
         commands: [],
       }),
     };
-    const result = await runDepositInBoxHost({
-      repositoryFullName: 'o/r', revision: 'main', branch: 'main', commit: null,
-      obfuscations: null, forcedExclusions: [], demandContext: [], hostFactory: async () => fakeHost,
+    await expect(
+      runDepositInBoxHost({
+        repositoryFullName: 'o/r',
+        revision: 'main',
+        branch: 'main',
+        commit: null,
+        obfuscations: null,
+        forcedExclusions: [],
+        demandContext: [],
+        hostFactory: async () => fakeHost,
+      }),
+    ).rejects.toThrow(/zero depositOptions/);
+  });
+});
+
+describe('formatDepositHostFailure', () => {
+  it('prefers failed command stderr and evidence.error', () => {
+    const msg = formatDepositHostFailure({
+      outcome: 'failed',
+      sandboxId: 'sbx1',
+      commands: [
+        { label: 'runtime-readiness', exitCode: 0, stderr: '', stdout: '' },
+        { label: 'asset-pack-pipeline-run', exitCode: 1, stderr: 'boom stack', stdout: '' },
+      ],
+      artifacts: {
+        evidence: { error: { name: 'Error', message: 'clone failed' }, resultReasons: ['no tree'] },
+      },
     });
-    expect(result.options).toEqual([]);
-    expect(result.outcome).toBe('completed');
+    expect(msg).toMatch(/asset-pack-pipeline-run/);
+    expect(msg).toMatch(/boom stack/);
+    expect(msg).toMatch(/clone failed/);
+    expect(msg).not.toMatch(/Validation absolutes/);
   });
 });
