diff --git a/packages/mobile-app/components/chat/Composer.tsx b/packages/mobile-app/components/chat/Composer.tsx index ca136d4c8d..99a924c798 100644 --- a/packages/mobile-app/components/chat/Composer.tsx +++ b/packages/mobile-app/components/chat/Composer.tsx @@ -1,19 +1,34 @@ import { IconArrowUp, IconAt, + IconBolt, + IconBulb, + IconCamera, + IconChevronDown, + IconChevronLeft, + IconChevronRight, + IconClock, IconFileText, IconMicrophone, IconPhoto, IconPlayerStopFilled, + IconPlugConnected, + IconPlus, IconRobot, + IconTools, + IconUpload, IconX, } from "@tabler/icons-react-native"; +import * as DocumentPicker from "expo-document-picker"; +import * as FileSystem from "expo-file-system"; import * as ImagePicker from "expo-image-picker"; import { useNavigation, useRouter } from "expo-router"; -import { useEffect, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ActivityIndicator, Image, + Modal, + Platform, Pressable, ScrollView, Text, @@ -35,6 +50,29 @@ import type { import type { AgentChatSettings } from "@/lib/agent-chat/use-agent-chat"; import { getAndClearLastDictatedText } from "@/lib/voice-api"; +export type ActionTag = { + id: string; + label: string; + icon: "bolt" | "bulb" | "clock" | "tools" | "upload"; +}; + +function renderActionTagIcon(iconName: ActionTag["icon"]) { + switch (iconName) { + case "bolt": + return ; + case "bulb": + return ; + case "clock": + return ; + case "tools": + return ; + case "upload": + return ; + default: + return null; + } +} + function MentionRowIcon({ refType }: { refType: string }) { if (refType === "agent" || refType === "custom-agent") { return ; @@ -46,33 +84,211 @@ function MentionRowIcon({ refType }: { refType: string }) { } function settingsSummary(settings: AgentChatSettings): string { - const model = settings.model ? settings.model.replace(/-\d{8}$/, "") : "Auto"; + if (!settings.model) return "Auto"; + const raw = settings.model.replace(/-\d{8}$/, ""); + let model = raw; + if (/sonnet/i.test(raw)) model = "Sonnet 5"; + else if (/opus/i.test(raw)) model = "Opus 3.5"; + else if (/haiku/i.test(raw)) model = "Haiku 3.5"; + else if (/gpt-4o/i.test(raw)) model = "GPT-4o"; + else if (/gemini/i.test(raw)) model = "Gemini 2.0"; + else { + model = raw.charAt(0).toUpperCase() + raw.slice(1); + } const effort = settings.effort ? ` · ${settings.effort[0]!.toUpperCase()}${settings.effort.slice(1, 3)}` : ""; return `${model}${effort}`; } -async function pickImageAttachment(): Promise { - const result = await ImagePicker.launchImageLibraryAsync({ - mediaTypes: ["images"], - quality: 0.7, - base64: true, - exif: false, - }); - const asset = result.assets?.[0]; - if (result.canceled || !asset?.base64) return null; - const mimeType = asset.mimeType?.startsWith("image/") - ? asset.mimeType - : "image/jpeg"; +function detectMimeType(fileName: string, providedMime?: string): string { + if ( + providedMime && + providedMime !== "application/octet-stream" && + providedMime !== "binary/octet-stream" + ) { + return providedMime; + } + const ext = fileName.split(".").pop()?.toLowerCase(); + switch (ext) { + case "jpg": + case "jpeg": + return "image/jpeg"; + case "png": + return "image/png"; + case "gif": + return "image/gif"; + case "webp": + return "image/webp"; + case "heic": + case "heif": + return "image/heic"; + case "svg": + return "image/svg+xml"; + case "pdf": + return "application/pdf"; + case "txt": + return "text/plain"; + case "json": + return "application/json"; + case "csv": + return "text/csv"; + case "md": + return "text/markdown"; + default: + return providedMime || "application/octet-stream"; + } +} + +async function getAssetDataUrl( + uri: string, + mimeType: string, + fileObj?: File | Blob, +): Promise { + if (fileObj && typeof FileReader !== "undefined") { + try { + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = reject; + reader.readAsDataURL(fileObj); + }); + } catch { + // fallback + } + } + + if (Platform.OS !== "web") { + try { + const base64 = await FileSystem.readAsStringAsync(uri, { + encoding: FileSystem.EncodingType.Base64, + }); + return `data:${mimeType};base64,${base64}`; + } catch (e) { + console.warn( + "FileSystem readAsStringAsync failed, trying fetch fallback:", + e, + ); + } + } + + try { + const res = await fetch(uri); + const blob = await res.blob(); + return await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = reject; + reader.readAsDataURL(blob); + }); + } catch (e) { + console.error("Failed to read asset data URL:", e); + return null; + } +} + +async function documentAssetToAttachment( + asset: DocumentPicker.DocumentPickerAsset, +): Promise { + const name = asset.name || "file"; + const mimeType = detectMimeType(name, asset.mimeType); + + const dataUrl = await getAssetDataUrl( + asset.uri, + mimeType, + (asset as { file?: File }).file, + ); + if (!dataUrl) return null; + + const isImage = mimeType.startsWith("image/"); + return { + type: isImage ? "image" : "file", + name, + contentType: mimeType, + data: dataUrl, + }; +} + +async function pickAnyFileAttachments(): Promise { + try { + const result = await DocumentPicker.getDocumentAsync({ + type: "*/*", + multiple: true, + copyToCacheDirectory: true, + }); + if (result.canceled || !result.assets?.length) return []; + const converted = await Promise.all( + result.assets.map(documentAssetToAttachment), + ); + return converted.filter((a): a is ChatAttachment => a !== null); + } catch (error) { + console.error("pickAnyFileAttachments error:", error); + return []; + } +} + +async function imageAssetToAttachment( + asset: ImagePicker.ImagePickerAsset, + fallbackName: string, +): Promise { + const name = asset.fileName ?? fallbackName; + const mimeType = detectMimeType(name, asset.mimeType ?? "image/jpeg"); + + let dataUrl: string | null = null; + if (asset.base64) { + dataUrl = `data:${mimeType};base64,${asset.base64}`; + } else if (asset.uri) { + dataUrl = await getAssetDataUrl(asset.uri, mimeType); + } + + if (!dataUrl) return null; + return { type: "image", - name: asset.fileName ?? "photo.jpg", + name, contentType: mimeType, - data: `data:${mimeType};base64,${asset.base64}`, + data: dataUrl, }; } +async function captureCameraAttachment(): Promise { + try { + const permission = await ImagePicker.requestCameraPermissionsAsync(); + if (!permission.granted) return null; + const result = await ImagePicker.launchCameraAsync({ + mediaTypes: ["images"], + quality: 0.7, + base64: true, + exif: false, + }); + const asset = result.assets?.[0]; + if (result.canceled || !asset) return null; + return await imageAssetToAttachment(asset, "camera_photo.jpg"); + } catch (error) { + console.error("captureCameraAttachment error:", error); + return null; + } +} + +async function pickPhotoFromLibrary(): Promise { + try { + const permission = await ImagePicker.requestMediaLibraryPermissionsAsync(); + if (!permission.granted) return null; + const result = await ImagePicker.launchImageLibraryAsync({ + mediaTypes: ["images"], + quality: 0.7, + base64: true, + exif: false, + }); + const asset = result.assets?.[0]; + if (result.canceled || !asset) return null; + return await imageAssetToAttachment(asset, "photo.jpg"); + } catch (error) { + console.error("pickPhotoFromLibrary error:", error); + return null; + } +} + export function Composer({ isStreaming, settings, @@ -84,7 +300,6 @@ export function Composer({ }: { isStreaming: boolean; settings: AgentChatSettings; - /** Active thread's app — mentions are fetched from this app. */ baseUrl?: string; onSend: ( text: string, @@ -102,8 +317,13 @@ export function Composer({ const [references, setReferences] = useState([]); const [mentionItems, setMentionItems] = useState([]); const [mentionLoading, setMentionLoading] = useState(false); + const [plusMenuOpen, setPlusMenuOpen] = useState(false); + const [menuScreen, setMenuScreen] = useState<"main" | "skill">("main"); + const [actionTag, setActionTag] = useState(null); + const canSend = - (text.trim().length > 0 || attachments.length > 0) && !isStreaming; + (text.trim().length > 0 || attachments.length > 0 || actionTag !== null) && + !isStreaming; // A mention is being typed only when the caret is a collapsed cursor. const activeMention = useMemo( @@ -169,8 +389,6 @@ export function Composer({ if (dictated) { setText((current) => { const next = current ? current + "\n" + dictated : dictated; - // Move the controlled caret to the end so the next keystrokes land - // after the dictated text, not before it. setSelection({ start: next.length, end: next.length }); return next; }); @@ -185,57 +403,150 @@ export function Composer({ const submit = () => { if (!canSend) return; - const value = text.trim(); - // Only send references still present in the text — a mention the user - // deleted should not silently travel with the turn. + const raw = text.trim(); + const value = actionTag + ? raw + ? `[${actionTag.label}] ${raw}` + : `Perform ${actionTag.label}` + : raw; + const activeReferences = references.filter((r) => value.includes(`@${r.name}`), ); setText(""); setAttachments([]); setReferences([]); + setActionTag(null); setSelection({ start: 0, end: 0 }); onSend(value, attachments, activeReferences); }; - const attach = () => { - void pickImageAttachment().then((attachment) => { - if (attachment) setAttachments((current) => [...current, attachment]); + const addAttachment = useCallback((attachment: ChatAttachment | null) => { + if (attachment) setAttachments((current) => [...current, attachment]); + }, []); + + const addAttachments = useCallback((incoming: ChatAttachment[]) => { + if (incoming.length) setAttachments((current) => [...current, ...incoming]); + }, []); + + useEffect(() => { + const recover = () => { + void ImagePicker.getPendingResultAsync() + .then(async (result) => { + if (!result || "code" in result) return; + if (result.canceled) return; + const asset = result.assets?.[0]; + if (!asset) return; + addAttachment(await imageAssetToAttachment(asset, "photo.jpg")); + }) + .catch(() => {}); + }; + recover(); + const unsubscribe = navigation.addListener("focus", recover); + return unsubscribe; + }, [navigation, addAttachment]); + + const pendingActionRef = useRef<(() => void) | null>(null); + const runPendingAction = () => { + const action = pendingActionRef.current; + pendingActionRef.current = null; + action?.(); + }; + const closeMenuThen = (action: () => void) => { + pendingActionRef.current = action; + setPlusMenuOpen(false); + if (Platform.OS !== "ios") setTimeout(runPendingAction, 300); + }; + + const handleOpenPlusMenu = () => { + setMenuScreen("main"); + setPlusMenuOpen(true); + }; + + const handleUploadFile = () => { + closeMenuThen(() => { + void pickAnyFileAttachments().then(addAttachments); + }); + }; + + const handleTakePhoto = () => { + closeMenuThen(() => { + void captureCameraAttachment().then(addAttachment); + }); + }; + + const handlePickPhoto = () => { + closeMenuThen(() => { + void pickPhotoFromLibrary().then(addAttachment); + }); + }; + + const handleSelectActionTag = (tag: ActionTag) => { + setActionTag(tag); + setPlusMenuOpen(false); + }; + + const handleUploadSkillFile = () => { + setActionTag({ + id: "upload-skill", + label: "Upload Skill File", + icon: "upload", + }); + closeMenuThen(() => { + void pickAnyFileAttachments().then(addAttachments); }); }; + const handleIntegrations = () => { + setPlusMenuOpen(false); + onOpenSettings(); + }; + return ( - + {attachments.length > 0 && ( - {attachments.map((attachment, index) => ( - - - - setAttachments((current) => - current.filter((_, i) => i !== index), - ) - } - accessibilityRole="button" - accessibilityLabel={`Remove ${attachment.name}`} + {attachments.map((attachment, index) => { + const isImage = + attachment.type === "image" || + attachment.contentType?.startsWith("image/"); + return ( + - - - - ))} + {isImage && attachment.data ? ( + + ) : ( + + + + )} + + + setAttachments((current) => + current.filter((_, i) => i !== index), + ) + } + accessibilityRole="button" + accessibilityLabel={`Remove ${attachment.name}`} + > + + + + ); + })} )} @@ -282,99 +593,406 @@ export function Composer({ )} - - - - - + + {actionTag && ( + + {renderActionTagIcon(actionTag.icon)} + + {actionTag.label} + + setActionTag(null)} + className="p-0.5 ml-1 active:opacity-75" + accessibilityRole="button" + accessibilityLabel={`Remove ${actionTag.label} tag`} + > + + + + )} + + + setSelection(event.nativeEvent.selection) + } + placeholder="Message the agent… (@ to mention)" + placeholderTextColor="#71717a" + multiline + keyboardAppearance="dark" + accessibilityLabel="Message input" + nativeID="chat-composer-input" + /> + + - + - - setSelection(event.nativeEvent.selection) - } - placeholder="Message the agent… (@ to mention)" - placeholderTextColor="#71717a" - multiline - keyboardAppearance="dark" - accessibilityLabel="Message input" - nativeID="chat-composer-input" - /> - {isStreaming ? ( + + - + + {settingsSummary(settings)} + + - ) : ( + - + + {settings.mode === "plan" ? "Plan" : "Act"} + + - )} - - - - - {settingsSummary(settings)} - - - · - - + )} + + - {settings.mode === "plan" ? "Plan" : "Act"} - - + + + + {isStreaming ? ( + + + + ) : ( + + + + )} + + + setPlusMenuOpen(false)} + onDismiss={runPendingAction} + > + setPlusMenuOpen(false)} + accessibilityLabel="Dismiss actions menu" + > + + {menuScreen === "main" ? ( + <> + + + Actions & Tools + + setPlusMenuOpen(false)} + className="p-1 active:opacity-75" + accessibilityRole="button" + accessibilityLabel="Close menu" + > + + + + + + + + + + + Choose Photo + + + Select an image from photo library + + + + + + + + + + + Upload File + + + Images, PDFs, text/code, JSON, CSV + + + + + + + + + + + Take Photo + + + Capture a photo with your camera + + + + + + handleSelectActionTag({ + id: "schedule-task", + label: "Schedule Task", + icon: "clock", + }) + } + accessibilityRole="button" + accessibilityLabel="Schedule Task" + > + + + + + + Schedule Task + + + Run something on a schedule + + + + + + handleSelectActionTag({ + id: "create-automation", + label: "Create Automation", + icon: "bolt", + }) + } + accessibilityRole="button" + accessibilityLabel="Create Automation" + > + + + + + + Create Automation + + + Set up a when-X-do-Y rule + + + + + + handleSelectActionTag({ + id: "create-extension", + label: "Create Extension", + icon: "tools", + }) + } + accessibilityRole="button" + accessibilityLabel="Create Extension" + > + + + + + + Create Extension + + + Build a mini app extension + + + + + + + + + + + Integrations + + + Connect MCP tools to the agent + + + + + setMenuScreen("skill")} + accessibilityRole="button" + accessibilityLabel="Create Skill" + > + + + + + + Create Skill + + + Teach the agent a new ability + + + + + + ) : ( + <> + + setMenuScreen("main")} + className="flex-row items-center gap-1 p-1 active:opacity-75" + accessibilityRole="button" + accessibilityLabel="Back to main menu" + > + + + Create Skill + + + setPlusMenuOpen(false)} + className="p-1 active:opacity-75" + accessibilityRole="button" + accessibilityLabel="Close menu" + > + + + + + + handleSelectActionTag({ + id: "create-skill", + label: "Create Skill", + icon: "bulb", + }) + } + accessibilityRole="button" + accessibilityLabel="Create new skill" + > + + + + + + Create new skill + + + Describe a skill and let the agent draft it + + + + + + + + + + + Upload skill file + + + Import an existing SKILL.md file + + + + + )} + + + ); } diff --git a/packages/mobile-app/package.json b/packages/mobile-app/package.json index b1a08b9791..bd4ca37b99 100644 --- a/packages/mobile-app/package.json +++ b/packages/mobile-app/package.json @@ -36,6 +36,7 @@ "expo-clipboard": "~57.0.1", "expo-constants": "~57.0.5", "expo-dev-client": "~57.0.6", + "expo-document-picker": "~57.0.1", "expo-file-system": "~57.0.1", "expo-font": "~57.0.1", "expo-haptics": "~57.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b4e6adacbe..ad1bdd727b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1652,6 +1652,9 @@ importers: expo-dev-client: specifier: ~57.0.6 version: 57.0.6(expo@57.0.6)(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7)) + expo-document-picker: + specifier: ~57.0.1 + version: 57.0.1(expo@57.0.6) expo-file-system: specifier: ~57.0.1 version: 57.0.1(expo@57.0.6)(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7)) @@ -15856,6 +15859,11 @@ packages: expo: '*' react-native: '*' + expo-document-picker@57.0.1: + resolution: {integrity: sha512-qBwM5oxDZ3I9kwFD3pUE1oK/WNv9artoEKO6UpqhQgNRr0XA1ALRVWYjkF4+ge9lUNDRehjTm/jenINkzqg84g==} + peerDependencies: + expo: '*' + expo-file-system@57.0.1: resolution: {integrity: sha512-w7/ERvQFrGP2apTO9lDtZ+O6JQIhfakL7+Xqzh+rfMO9B4LB4qwrz+YvLgir8KFRVX64JHBnRuYBVLY1oQZcqw==} peerDependencies: @@ -31838,6 +31846,10 @@ snapshots: expo-dev-menu-interface: 57.0.0(expo@57.0.6) react-native: 0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7) + expo-document-picker@57.0.1(expo@57.0.6): + dependencies: + expo: 57.0.6(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.5)(@typescript/typescript6@6.0.2)(expo-router@57.0.6)(react-dom@19.2.7(react@19.2.7))(react-native-webview@13.16.1(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7))(react@19.2.7))(react-native-worklets@0.11.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7))(react@19.2.7))(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7))(react@19.2.7) + expo-file-system@57.0.1(expo@57.0.6)(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7)): dependencies: expo: 57.0.6(@babel/core@7.29.7)(@expo/dom-webview@57.0.1)(@expo/metro-runtime@57.0.5)(@typescript/typescript6@6.0.2)(expo-router@57.0.6)(react-dom@19.2.7(react@19.2.7))(react-native-webview@13.16.1(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7))(react@19.2.7))(react-native-worklets@0.11.1(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7))(react@19.2.7))(react-native@0.86.0(@babel/core@7.29.7)(@react-native/metro-config@0.86.0(@babel/core@7.29.7))(@types/react@19.2.17)(react@19.2.7))(react@19.2.7)