fix(mobile): prepare cloud attachments before sending (port #3838) (#3866)

Author: Gilbert09

apps/mobile/src/features/tasks/composer/TaskChatComposer.tsx

@@ -21,9 +21,10 @@ import {
   Stop,
 } from "phosphor-react-native";
 import { useFeatureFlag } from "posthog-react-native";
-import { useCallback, useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import {
   ActivityIndicator,
+  Alert,
   Keyboard,
   Pressable,
   ScrollView,
@@ -37,7 +38,11 @@ import { useThemeColors } from "@/lib/theme";
 import type { MessagingMode } from "../stores/messagingModeStore";
 import { AgentConfigControls } from "./AgentConfigControls";
 import { AttachmentSheet } from "./attachments/AttachmentSheet";
-import { AttachmentsBar } from "./attachments/AttachmentsBar";
+import {
+  type AttachmentStatus,
+  AttachmentsBar,
+} from "./attachments/AttachmentsBar";
+import { attachmentPreparer } from "./attachments/buildCloudPrompt";
 import {
   captureFromCamera,
   pickDocument,
@@ -122,6 +127,9 @@ export function TaskChatComposer({
   const modelConfigOption = getModelConfigOption(configOptions);
   const [message, setMessage] = useState(() => initialMessage ?? "");
   const [attachments, setAttachments] = useState<PendingAttachment[]>([]);
+  const [attachmentStatus, setAttachmentStatus] = useState<
+    Record<string, AttachmentStatus>
+  >({});
   const [attachmentSheetOpen, setAttachmentSheetOpen] = useState(false);
 
   // Mirror composer state into refs so a failed send can read the current
@@ -132,6 +140,48 @@ export function TaskChatComposer({
   attachmentsRef.current = attachments;
   const submissionRef = useRef(0);
 
+  const clearStatus = useCallback((id: string) => {
+    setAttachmentStatus((prev) => {
+      if (!(id in prev)) return prev;
+      const { [id]: _dropped, ...rest } = prev;
+      return rest;
+    });
+  }, []);
+
+  // Encode eagerly so oversized/unsupported files fail at attach, not send.
+  const beginPreparing = useCallback(
+    (att: PendingAttachment) => {
+      setAttachmentStatus((prev) => ({ ...prev, [att.id]: "preparing" }));
+      attachmentPreparer.prepare(att).then(
+        () => {
+          if (attachmentsRef.current.some((a) => a.id === att.id)) {
+            clearStatus(att.id);
+          }
+        },
+        (error: unknown) => {
+          if (!attachmentsRef.current.some((a) => a.id === att.id)) return;
+          setAttachmentStatus((prev) => ({ ...prev, [att.id]: "error" }));
+          Alert.alert(
+            "Attachment can't be sent",
+            error instanceof Error
+              ? error.message
+              : "This file couldn't be prepared. Remove it and try another.",
+          );
+        },
+      );
+    },
+    [clearStatus],
+  );
+
+  const loadAttachments = useCallback(
+    (next: PendingAttachment[]) => {
+      setAttachments(next);
+      setAttachmentStatus({});
+      for (const att of next) beginPreparing(att);
+    },
+    [beginPreparing],
+  );
+
   useEffect(() => {
     if (!initialMessage) return;
     setMessage(initialMessage);
@@ -140,8 +190,17 @@ export function TaskChatComposer({
   useEffect(() => {
     if (!restoredDraft) return;
     setMessage(restoredDraft.text);
-    setAttachments(restoredDraft.attachments);
-  }, [restoredDraft]);
+    loadAttachments(restoredDraft.attachments);
+  }, [restoredDraft, loadAttachments]);
+
+  useEffect(
+    () => () => {
+      for (const att of attachmentsRef.current) {
+        attachmentPreparer.forget(att.id);
+      }
+    },
+    [],
+  );
 
   useEffect(() => {
     if (!hasLiveConfig) return;
@@ -174,9 +233,12 @@ export function TaskChatComposer({
   const isTranscribing = status === "transcribing";
 
   const hasContent = !isComposerEmpty({ text: message, attachments });
+  const statuses = Object.values(attachmentStatus);
+  const attachmentsPreparing = statuses.includes("preparing");
+  const sendBlocked = attachmentsPreparing || statuses.includes("error");
   const primaryAction = resolveComposerPrimaryAction({
     hasContent,
-    disabled,
+    disabled: disabled || sendBlocked,
     isRecording,
     isTranscribing,
     canStop: !isUserTurn && !!onStop,
@@ -187,7 +249,7 @@ export function TaskChatComposer({
 
   const applyContent = (content: ComposerContent) => {
     setMessage(content.text);
-    setAttachments(content.attachments);
+    loadAttachments(content.attachments);
   };
 
   const handleSend = () => {
@@ -214,14 +276,19 @@ export function TaskChatComposer({
   ) => {
     try {
       const att = await picker();
-      if (att) setAttachments((prev) => [...prev, att]);
+      if (att) {
+        setAttachments((prev) => [...prev, att]);
+        beginPreparing(att);
+      }
     } catch (err) {
       log.error("Failed to pick attachment", err);
     }
   };
 
   const removeAttachment = (id: string) => {
     setAttachments((prev) => prev.filter((a) => a.id !== id));
+    attachmentPreparer.forget(id);
+    clearStatus(id);
   };
 
   const handleMicPress = async () => {
@@ -282,6 +349,7 @@ export function TaskChatComposer({
             <AttachmentsBar
               attachments={attachments}
               onRemove={removeAttachment}
+              statuses={attachmentStatus}
             />
             <TextInput
               className="px-4 pt-3.5 pb-3 text-[15px] text-gray-12"
@@ -368,20 +436,22 @@ export function TaskChatComposer({
                   canSend ? handleSend : showStop ? handleStop : handleMicPress
                 }
                 onLongPress={handleMicLongPress}
-                disabled={isTranscribing || disabled}
+                disabled={isTranscribing || disabled || sendBlocked}
                 className={`h-9 w-9 items-center justify-center rounded-lg ${
                   canSend ? "bg-gray-12" : "bg-gray-3"
                 }`}
               >
-                {isTranscribing ? (
+                {isTranscribing || attachmentsPreparing ? (
                   <ActivityIndicator
                     size="small"
                     color={themeColors.gray[12]}
                   />
-                ) : canSend ? (
+                ) : canSend || sendBlocked ? (
                   <ArrowUp
                     size={18}
-                    color={themeColors.background}
+                    color={
+                      canSend ? themeColors.background : themeColors.gray[9]
+                    }
                     weight="bold"
                   />
                 ) : isRecording || showStop ? (

apps/mobile/src/features/tasks/composer/attachments/AttachmentsBar.tsx

@@ -1,12 +1,21 @@
 import { Text } from "@components/text";
-import { FileText, X } from "phosphor-react-native";
-import { Image, Pressable, ScrollView, View } from "react-native";
+import { FileText, WarningCircle, X } from "phosphor-react-native";
+import {
+  ActivityIndicator,
+  Image,
+  Pressable,
+  ScrollView,
+  View,
+} from "react-native";
 import { useThemeColors } from "@/lib/theme";
 import type { PendingAttachment } from "./types";
 
+export type AttachmentStatus = "preparing" | "error";
+
 interface AttachmentsBarProps {
   attachments: PendingAttachment[];
   onRemove: (id: string) => void;
+  statuses?: Record<string, AttachmentStatus>;
 }
 
 function truncate(name: string, max = 18): string {
@@ -18,7 +27,36 @@ function truncate(name: string, max = 18): string {
   return `${name.slice(0, max - 1)}…`;
 }
 
-export function AttachmentsBar({ attachments, onRemove }: AttachmentsBarProps) {
+function StatusOverlay({ status }: { status?: AttachmentStatus }) {
+  const themeColors = useThemeColors();
+  if (!status) return null;
+  return (
+    <View
+      className="absolute inset-0 items-center justify-center rounded-lg bg-gray-1/70"
+      accessibilityLabel={
+        status === "preparing"
+          ? "Preparing attachment"
+          : "Attachment failed to prepare"
+      }
+    >
+      {status === "preparing" ? (
+        <ActivityIndicator size="small" color={themeColors.gray[12]} />
+      ) : (
+        <WarningCircle
+          size={20}
+          color={themeColors.status.error}
+          weight="fill"
+        />
+      )}
+    </View>
+  );
+}
+
+export function AttachmentsBar({
+  attachments,
+  onRemove,
+  statuses,
+}: AttachmentsBarProps) {
   const themeColors = useThemeColors();
   if (attachments.length === 0) return null;
 
@@ -60,6 +98,7 @@ export function AttachmentsBar({ attachments, onRemove }: AttachmentsBarProps) {
               </Text>
             </View>
           )}
+          <StatusOverlay status={statuses?.[att.id]} />
           <Pressable
             onPress={() => onRemove(att.id)}
             hitSlop={8}

apps/mobile/src/features/tasks/composer/attachments/attachmentPreparer.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, it, vi } from "vitest";
+import { createAttachmentPreparer } from "./attachmentPreparer";
+import type { CloudPromptBlock, PendingAttachment } from "./types";
+
+function attachment(id: string): PendingAttachment {
+  return {
+    kind: "document",
+    id,
+    uri: `file://${id}.txt`,
+    fileName: `${id}.txt`,
+    mimeType: "text/plain",
+  };
+}
+
+function block(id: string): CloudPromptBlock {
+  return { type: "text", text: id };
+}
+
+describe("createAttachmentPreparer", () => {
+  it("caches a resolved block and reuses it across prepares", async () => {
+    const build = vi.fn(async (att: PendingAttachment) => block(att.id));
+    const preparer = createAttachmentPreparer(build);
+
+    const first = await preparer.prepare(attachment("a"));
+    const second = await preparer.prepare(attachment("a"));
+
+    expect(first).toEqual(block("a"));
+    expect(second).toBe(first);
+    expect(build).toHaveBeenCalledTimes(1);
+  });
+
+  it("dedupes concurrent preparation of the same attachment", async () => {
+    const build = vi.fn(async (att: PendingAttachment) => block(att.id));
+    const preparer = createAttachmentPreparer(build);
+
+    const [first, second] = await Promise.all([
+      preparer.prepare(attachment("a")),
+      preparer.prepare(attachment("a")),
+    ]);
+
+    expect(first).toBe(second);
+    expect(build).toHaveBeenCalledTimes(1);
+  });
+
+  it("prepares distinct attachments independently", async () => {
+    const build = vi.fn(async (att: PendingAttachment) => block(att.id));
+    const preparer = createAttachmentPreparer(build);
+
+    expect(await preparer.prepare(attachment("a"))).toEqual(block("a"));
+    expect(await preparer.prepare(attachment("b"))).toEqual(block("b"));
+    expect(build).toHaveBeenCalledTimes(2);
+  });
+
+  it("evicts a failed preparation so it can be retried", async () => {
+    const build = vi
+      .fn<(att: PendingAttachment) => Promise<CloudPromptBlock>>()
+      .mockRejectedValueOnce(new Error("too large"))
+      .mockImplementation(async (att) => block(att.id));
+    const preparer = createAttachmentPreparer(build);
+
+    await expect(preparer.prepare(attachment("a"))).rejects.toThrow(
+      "too large",
+    );
+    expect(await preparer.prepare(attachment("a"))).toEqual(block("a"));
+    expect(build).toHaveBeenCalledTimes(2);
+  });
+
+  it("re-reads after forget", async () => {
+    const build = vi.fn(async (att: PendingAttachment) => block(att.id));
+    const preparer = createAttachmentPreparer(build);
+
+    await preparer.prepare(attachment("a"));
+    preparer.forget("a");
+    await preparer.prepare(attachment("a"));
+
+    expect(build).toHaveBeenCalledTimes(2);
+  });
+});

apps/mobile/src/features/tasks/composer/attachments/attachmentPreparer.ts

@@ -0,0 +1,29 @@
+import type { CloudPromptBlock, PendingAttachment } from "./types";
+
+export interface AttachmentPreparer {
+  prepare(attachment: PendingAttachment): Promise<CloudPromptBlock>;
+  forget(id: string): void;
+}
+
+export function createAttachmentPreparer(
+  build: (attachment: PendingAttachment) => Promise<CloudPromptBlock>,
+): AttachmentPreparer {
+  const cache = new Map<string, Promise<CloudPromptBlock>>();
+
+  return {
+    prepare(attachment) {
+      const existing = cache.get(attachment.id);
+      if (existing) return existing;
+
+      const pending = build(attachment).catch((error) => {
+        cache.delete(attachment.id);
+        throw error;
+      });
+      cache.set(attachment.id, pending);
+      return pending;
+    },
+    forget(id) {
+      cache.delete(id);
+    },
+  };
+}

apps/mobile/src/features/tasks/composer/attachments/buildCloudPrompt.ts

@@ -1,4 +1,5 @@
 import * as FileSystem from "expo-file-system/legacy";
+import { createAttachmentPreparer } from "./attachmentPreparer";
 import type { CloudPromptBlock, PendingAttachment } from "./types";
 
 const MAX_EMBEDDED_TEXT_CHARS = 100_000;
@@ -92,7 +93,9 @@ function estimateBase64Bytes(base64: string): number {
   return Math.floor((base64.length * 3) / 4) - padding;
 }
 
-async function buildBlock(att: PendingAttachment): Promise<CloudPromptBlock> {
+export async function buildAttachmentBlock(
+  att: PendingAttachment,
+): Promise<CloudPromptBlock> {
   if (att.kind === "image") {
     const base64 = await FileSystem.readAsStringAsync(att.uri, {
       encoding: FileSystem.EncodingType.Base64,
@@ -129,6 +132,9 @@ async function buildBlock(att: PendingAttachment): Promise<CloudPromptBlock> {
   };
 }
 
+export const attachmentPreparer =
+  createAttachmentPreparer(buildAttachmentBlock);
+
 /**
  * Reads each attachment from disk and assembles the cloud-prompt block array
  * the agent server expects. Throws if any individual attachment fails so the
@@ -142,7 +148,9 @@ export async function buildCloudPromptBlocks(
   const trimmed = text.trim();
   if (trimmed) blocks.push({ type: "text", text: trimmed });
   for (const attachment of attachments) {
-    blocks.push(await buildBlock(attachment));
+    blocks.push(await attachmentPreparer.prepare(attachment));
+    // Base64/text payloads are large; release once folded into the prompt.
+    attachmentPreparer.forget(attachment.id);
   }
   if (blocks.length === 0) {
     throw new Error("Cloud prompt cannot be empty");