From 67bc4dc523afe0c40911ed5c9e672ab8101781b4 Mon Sep 17 00:00:00 2001 From: Jedr Blaszyk Date: Wed, 18 Mar 2026 20:45:48 -0700 Subject: [PATCH 1/4] feat: live agent chat with client-side document tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add EditorBridge that connects AI agent tools to a live DocxEditor instance. The agent can read the document, add comments, suggest tracked changes, and scroll — all executing on the client without reloading the document. Key changes: - DocxEditorRef: 6 new methods (addComment, replyToComment, resolveComment, proposeReplacement, scrollToIndex, getComments) - EditorBridge (packages/agent-use/bridge.ts): wraps DocxEditorRef for agent use - Tool definitions (packages/agent-use/tools/): 6 tools with OpenAI-compatible schemas (read_document, read_comments, read_changes, add_comment, suggest_replacement, scroll_to) - useAgentChat hook: wires tools to editor ref - agent-chat-demo: Next.js example with chat panel beside the editor Co-Authored-By: Claude Opus 4.6 (1M context) --- bun.lock | 10 +- examples/agent-chat-demo/.env.example | 5 + .../agent-chat-demo/app/api/chat/route.ts | 62 ++ examples/agent-chat-demo/app/globals.css | 3 + examples/agent-chat-demo/app/layout.tsx | 17 + examples/agent-chat-demo/app/page.tsx | 657 ++++++++++++++++++ examples/agent-chat-demo/next.config.ts | 7 + examples/agent-chat-demo/package.json | 24 + examples/agent-chat-demo/postcss.config.js | 3 + examples/agent-chat-demo/tsconfig.json | 27 + packages/agent-use/src/bridge.ts | 222 +++++- packages/agent-use/src/index.ts | 4 + packages/agent-use/src/tools/index.ts | 204 ++++++ packages/agent-use/src/tools/types.ts | 22 + packages/agent-use/src/useAgentChat.ts | 53 ++ packages/agent-use/tsup.config.ts | 2 +- packages/react/src/components/DocxEditor.tsx | 199 ++++++ specs/live-agent-chat.md | 619 +++++++++++++++++ 18 files changed, 2129 insertions(+), 11 deletions(-) create mode 100644 examples/agent-chat-demo/.env.example create mode 100644 examples/agent-chat-demo/app/api/chat/route.ts create mode 100644 examples/agent-chat-demo/app/globals.css create mode 100644 examples/agent-chat-demo/app/layout.tsx create mode 100644 examples/agent-chat-demo/app/page.tsx create mode 100644 examples/agent-chat-demo/next.config.ts create mode 100644 examples/agent-chat-demo/package.json create mode 100644 examples/agent-chat-demo/postcss.config.js create mode 100644 examples/agent-chat-demo/tsconfig.json create mode 100644 packages/agent-use/src/tools/index.ts create mode 100644 packages/agent-use/src/tools/types.ts create mode 100644 packages/agent-use/src/useAgentChat.ts create mode 100644 specs/live-agent-chat.md diff --git a/bun.lock b/bun.lock index 5cfb497b..b766c7db 100644 --- a/bun.lock +++ b/bun.lock @@ -38,14 +38,14 @@ }, "packages/agent-use": { "name": "@eigenpal/docx-editor-agents", - "version": "0.0.1", - "peerDependencies": { - "@eigenpal/docx-core": ">=0.0.1", + "version": "0.0.28", + "dependencies": { + "@eigenpal/docx-core": "workspace:*", }, }, "packages/core": { "name": "@eigenpal/docx-core", - "version": "0.0.1", + "version": "0.0.28", "bin": { "docx-editor-mcp": "./dist/mcp-cli.js", }, @@ -66,7 +66,7 @@ }, "packages/react": { "name": "@eigenpal/docx-js-editor", - "version": "0.0.27", + "version": "0.0.28", "dependencies": { "@radix-ui/react-select": "^2.2.6", "clsx": "^2.1.0", diff --git a/examples/agent-chat-demo/.env.example b/examples/agent-chat-demo/.env.example new file mode 100644 index 00000000..ad2e4794 --- /dev/null +++ b/examples/agent-chat-demo/.env.example @@ -0,0 +1,5 @@ +# Get your API key at https://platform.openai.com/api-keys +OPENAI_API_KEY=sk-... + +# Optional: override the model (default: gpt-4o) +# OPENAI_MODEL=gpt-4o diff --git a/examples/agent-chat-demo/app/api/chat/route.ts b/examples/agent-chat-demo/app/api/chat/route.ts new file mode 100644 index 00000000..2db6d804 --- /dev/null +++ b/examples/agent-chat-demo/app/api/chat/route.ts @@ -0,0 +1,62 @@ +/** + * Chat API route — thin proxy to OpenAI. + * + * Does NOT touch the document. Tool definitions are passed to OpenAI, + * but tool execution happens on the client via the EditorBridge. + * + * Flow: + * 1. Client sends { messages, tools } to this route + * 2. Route calls OpenAI with the tools + * 3. If OpenAI returns tool_calls, route returns them to the client + * 4. Client executes tools via EditorBridge, sends results back + * 5. Repeat until OpenAI returns text + */ + +import { NextRequest, NextResponse } from 'next/server'; +import OpenAI from 'openai'; +import type { + ChatCompletionMessageParam, + ChatCompletionTool, +} from 'openai/resources/chat/completions'; + +function getClient() { + return new OpenAI(); +} +const model = process.env.OPENAI_MODEL || 'gpt-4o'; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const { messages, tools } = body as { + messages: ChatCompletionMessageParam[]; + tools: ChatCompletionTool[]; + }; + + if (!messages || messages.length === 0) { + return NextResponse.json({ error: 'No messages provided' }, { status: 400 }); + } + + const openai = getClient(); + const response = await openai.chat.completions.create({ + model, + messages, + tools: tools && tools.length > 0 ? tools : undefined, + }); + + const choice = response.choices[0]; + if (!choice) { + return NextResponse.json({ error: 'Empty response from AI' }, { status: 502 }); + } + + return NextResponse.json({ + message: choice.message, + finishReason: choice.finish_reason, + }); + } catch (err) { + console.error('Chat API error:', err); + return NextResponse.json( + { error: err instanceof Error ? err.message : 'Internal error' }, + { status: 500 } + ); + } +} diff --git a/examples/agent-chat-demo/app/globals.css b/examples/agent-chat-demo/app/globals.css new file mode 100644 index 00000000..83a2032b --- /dev/null +++ b/examples/agent-chat-demo/app/globals.css @@ -0,0 +1,3 @@ +/* Import editor styles (CSS variables, toolbar layout, etc.) + In standalone usage: @import '@eigenpal/docx-js-editor/styles.css'; */ +@import '../../../packages/react/src/styles/editor.css'; diff --git a/examples/agent-chat-demo/app/layout.tsx b/examples/agent-chat-demo/app/layout.tsx new file mode 100644 index 00000000..babb8e1f --- /dev/null +++ b/examples/agent-chat-demo/app/layout.tsx @@ -0,0 +1,17 @@ +import type { Metadata } from 'next'; +import './globals.css'; + +export const metadata: Metadata = { + title: 'Chat with your Doc', + description: 'Upload a DOCX and chat with AI — it can add comments and suggest changes live', +}; + +export default function RootLayout({ children }: { children: React.ReactNode }) { + return ( + + + {children} + + + ); +} diff --git a/examples/agent-chat-demo/app/page.tsx b/examples/agent-chat-demo/app/page.tsx new file mode 100644 index 00000000..8517ff05 --- /dev/null +++ b/examples/agent-chat-demo/app/page.tsx @@ -0,0 +1,657 @@ +'use client'; + +import { useState, useRef, useCallback, useEffect } from 'react'; +import { DocxEditor, type DocxEditorRef } from '@eigenpal/docx-js-editor'; +import { + createEditorBridge, + getToolSchemas, + executeToolCall, + type EditorRefLike, +} from '@eigenpal/docx-editor-agents/bridge'; + +// ── Types ─────────────────────────────────────────────────────────────────── + +interface ChatMessage { + id: string; + role: 'user' | 'assistant'; + content: string; + toolCalls?: ToolCallLog[]; +} + +interface ToolCallLog { + name: string; + input: Record; + result: string; +} + +// Full OpenAI message for multi-turn context (keeps tool_calls + tool results) +// eslint-disable-next-line @typescript-eslint/no-explicit-any +type OpenAIMessage = any; + +// ── Helpers ───────────────────────────────────────────────────────────────── + +let msgId = 0; +function nextId() { + return `msg-${++msgId}`; +} + +const TOOL_LABELS: Record = { + read_document: 'Read document', + read_comments: 'Read comments', + read_changes: 'Read tracked changes', + add_comment: 'Add comment', + suggest_replacement: 'Suggest change', + scroll_to: 'Scroll to', +}; + +const SYSTEM_PROMPT = `You are a helpful document assistant. The user has a DOCX document open and is chatting with you about it. + +You have tools to: +- READ the document content (always do this first if you haven't seen the document yet) +- ADD COMMENTS to specific paragraphs +- SUGGEST REPLACEMENTS as tracked changes the user can accept/reject +- SCROLL to specific paragraphs + +Guidelines: +- Always read the document before making changes +- When adding comments or suggesting changes, reference the paragraph index [N] from read_document +- Keep comments concise and actionable +- For replacements, use a short search phrase (3-8 words) that uniquely identifies the text +- You can make multiple tool calls in a single turn +- After making changes, briefly tell the user what you did`; + +// ── Main Component ────────────────────────────────────────────────────────── + +export default function Home() { + const [documentBuffer, setDocumentBuffer] = useState(null); + const [documentName, setDocumentName] = useState(''); + const [messages, setMessages] = useState([]); + const [input, setInput] = useState(''); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(null); + const [dragOver, setDragOver] = useState(false); + const [expandedTools, setExpandedTools] = useState>(new Set()); + + const editorRef = useRef(null); + const fileInputRef = useRef(null); + const chatEndRef = useRef(null); + + // Full OpenAI message history — preserved across turns (includes tool_calls + tool results) + const openaiHistoryRef = useRef([]); + + // Auto-scroll chat + useEffect(() => { + chatEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [messages, isLoading]); + + // ── File handling ───────────────────────────────────────────────────────── + + const handleFile = useCallback((f: File) => { + if (!f.name.endsWith('.docx')) { + setError('Please upload a .docx file'); + return; + } + setError(null); + setDocumentName(f.name); + f.arrayBuffer().then((buf) => { + setDocumentBuffer(buf); + setMessages([]); + openaiHistoryRef.current = []; + }); + }, []); + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + setDragOver(false); + const f = e.dataTransfer.files[0]; + if (f) handleFile(f); + }, + [handleFile] + ); + + // ── Chat with client-side tool execution ────────────────────────────────── + + const sendMessage = async () => { + const text = input.trim(); + if (!text || !editorRef.current || isLoading) return; + + const userMsg: ChatMessage = { id: nextId(), role: 'user', content: text }; + setMessages((prev) => [...prev, userMsg]); + setInput(''); + setIsLoading(true); + setError(null); + + try { + // Append user message to persistent OpenAI history + openaiHistoryRef.current.push({ role: 'user', content: text }); + + const tools = getToolSchemas(); + const allToolCalls: ToolCallLog[] = []; + const bridge = createEditorBridge(editorRef.current as unknown as EditorRefLike, 'Assistant'); + + // Tool-use loop — call API, execute tools locally, repeat + const MAX_ITERATIONS = 10; + for (let i = 0; i < MAX_ITERATIONS; i++) { + const response = await fetch('/api/chat', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + messages: [{ role: 'system', content: SYSTEM_PROMPT }, ...openaiHistoryRef.current], + tools, + }), + }); + + if (!response.ok) { + const err = await response.json(); + throw new Error(err.error || 'Request failed'); + } + + const data = await response.json(); + const msg = data.message; + + // No tool calls — we're done, show the text response + if (!msg.tool_calls || msg.tool_calls.length === 0) { + openaiHistoryRef.current.push({ role: 'assistant', content: msg.content || '' }); + const assistantMsg: ChatMessage = { + id: nextId(), + role: 'assistant', + content: msg.content || '', + toolCalls: allToolCalls.length > 0 ? allToolCalls : undefined, + }; + setMessages((prev) => [...prev, assistantMsg]); + break; + } + + // Execute tool calls on the client via EditorBridge + openaiHistoryRef.current.push(msg); + + for (const toolCall of msg.tool_calls) { + let args: Record; + try { + args = JSON.parse(toolCall.function.arguments); + } catch { + args = {}; + } + const result = executeToolCall(toolCall.function.name, args, bridge); + + const resultStr = + typeof result.data === 'string' + ? result.data + : result.error || JSON.stringify(result.data); + + allToolCalls.push({ + name: toolCall.function.name, + input: args as Record, + result: resultStr, + }); + + // Append tool result to persistent history + openaiHistoryRef.current.push({ + role: 'tool', + tool_call_id: toolCall.id, + content: resultStr, + }); + } + } + } catch (err) { + setError(err instanceof Error ? err.message : 'Something went wrong'); + } finally { + setIsLoading(false); + } + }; + + const handleKeyDown = (e: React.KeyboardEvent) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + sendMessage(); + } + }; + + const toggleToolExpand = (id: string) => { + setExpandedTools((prev) => { + const next = new Set(prev); + if (next.has(id)) next.delete(id); + else next.add(id); + return next; + }); + }; + + // ── Upload screen ───────────────────────────────────────────────────────── + + if (!documentBuffer) { + return ( +
+
+
💬
+

Chat with your Doc

+

+ Upload a DOCX file and have a conversation with AI about it. The assistant can read your + document, add comments, and suggest changes — all live in the editor, no reloads. +

+ +
{ + e.preventDefault(); + setDragOver(true); + }} + onDragLeave={() => setDragOver(false)} + onDrop={handleDrop} + onClick={() => fileInputRef.current?.click()} + > + { + const f = e.target.files?.[0]; + if (f) handleFile(f); + }} + /> +
📄
+
Drop your DOCX here
+
or click to browse
+
+ + {error &&
{error}
} + + +
+
+ ); + } + + // ── Main layout: editor + chat ──────────────────────────────────────────── + + return ( +
+ {/* Header */} +
+
+ 💬 + {documentName} +
+ +
+ +
+ {/* Editor */} +
+ +
+ + {/* Chat panel */} +
+ {/* Messages */} +
+ {messages.length === 0 && ( +
+
💬
+
Ask anything about your document.
+
+ Try: "Review this for grammar issues" or "Summarize the key + points" +
+
+ )} + + {messages.map((msg) => ( +
+
+
{msg.content}
+
+ + {msg.toolCalls && msg.toolCalls.length > 0 && ( +
+ {msg.toolCalls.map((tc, i) => { + const tcId = `${msg.id}-tool-${i}`; + const isExpanded = expandedTools.has(tcId); + const isWrite = ['add_comment', 'suggest_replacement'].includes(tc.name); + return ( +
+
toggleToolExpand(tcId)}> + + {isWrite ? '\u270E' : '\u{1F50D}'} + + + {TOOL_LABELS[tc.name] || tc.name} + + {tc.name === 'add_comment' && tc.input.text && ( + + {' '} + — "{String(tc.input.text).slice(0, 50)} + {String(tc.input.text).length > 50 ? '...' : ''}" + + )} + {tc.name === 'suggest_replacement' && ( + + {' '} + — "{String(tc.input.search)}" → " + {String(tc.input.replaceWith)}" + + )} + + {isExpanded ? '\u25B2' : '\u25BC'} + +
+ {isExpanded && ( +
+
+ Input: + 
+                                  {JSON.stringify(tc.input, null, 2)}
+
+
+
+ Result: + 
+                                  {tc.result.length > 500
+                                    ? tc.result.slice(0, 500) + '...'
+                                    : tc.result}
+
+
+
+ )} +
+ ); + })} +
+ )} +
+ ))} + + {isLoading && ( +
+
+
+ + + +
+
+
+ )} + + {error && ( +
+
{error}
+
+ )} + +
+
+ + {/* Input */} +
+