diff --git a/packages/extension/src/agent/EventRecorder.background.ts b/packages/extension/src/agent/EventRecorder.background.ts new file mode 100644 index 000000000..9a9460c70 --- /dev/null +++ b/packages/extension/src/agent/EventRecorder.background.ts @@ -0,0 +1,397 @@ +/** + * EventRecorder — Background Script side + * + * Manages recording state, broadcasts start/stop to all content scripts, + * aggregates events from multiple tabs, monitors tab lifecycle, + * and forwards events to the sidepanel. + * + * Phase 6 enhancements: + * - Robust SW restart recovery with timestamp validation + * - Inject content script into newly opened tabs during recording + * - Dedup navigate events from tab onUpdated + * - Error handling for all chrome API calls + */ + +import type { RawRecordingEvent } from '@/lib/recording-types' + +const DEBUG_PREFIX = '[EventRecorder.background]' + +let isRecording = false +let recordingStartTime = 0 +let recordingSeqId = 0 // monotonic counter to detect stale operations +let tabIndexMap = new Map() // tabId → tabIdx +let lastNavigateUrl = new Map() // tabId → last navigated URL (dedup) + +// ─── Broadcast to Content Scripts ────────────────────────────────── + +async function broadcastToContentScripts(message: any) { + try { + const tabs = await chrome.tabs.query({}) + const promises = tabs + .filter( + (tab) => + tab.id && + tab.url && + !tab.url.startsWith('chrome://') && + !tab.url.startsWith('chrome-extension://') && + !tab.url.startsWith('about:') + ) + .map((tab) => + chrome.tabs.sendMessage(tab.id!, message).catch(() => { + // Content script may not be loaded yet, ignore + }) + ) + await Promise.allSettled(promises) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Broadcast error:', err) + } +} + +// ─── Tab Index Management ────────────────────────────────────────── + +async function refreshTabIndexMap() { + try { + const tabs = await chrome.tabs.query({ currentWindow: true }) + tabIndexMap.clear() + tabs.forEach((tab, idx) => { + if (tab.id) tabIndexMap.set(tab.id, idx) + }) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Failed to refresh tab index map:', err) + } +} + +function getTabIdx(tabId: number): number { + return tabIndexMap.get(tabId) ?? 0 +} + +// ─── Forward Event to Sidepanel ──────────────────────────────────── + +function forwardToSidepanel(event: RawRecordingEvent) { + chrome.runtime + .sendMessage({ + type: 'RECORDING_CONTROL', + action: 'recording_event', + payload: event, + }) + .catch(() => { + // Sidepanel may not be open, ignore + }) +} + +// ─── Inject Content Script into Tab ──────────────────────────────── + +async function ensureContentScriptInTab(tabId: number) { + try { + // Try to start recording directly; if content script is loaded this will work + await chrome.tabs.sendMessage(tabId, { + type: 'RECORDING_CONTROL', + action: 'status', + payload: { isRecording: true }, + }) + } catch { + // Content script not loaded — try scripting API injection + try { + await chrome.scripting.executeScript({ + target: { tabId }, + func: () => { + // The content script will be injected by the extension framework + // This is a no-op placeholder; the real content script auto-injects via manifest + }, + }) + // Give a brief delay then retry + setTimeout(() => { + chrome.tabs + .sendMessage(tabId, { + type: 'RECORDING_CONTROL', + action: 'start', + }) + .catch(() => { + console.debug(DEBUG_PREFIX, `Failed to start recording in tab ${tabId} after injection`) + }) + }, 500) + } catch (err) { + console.debug(DEBUG_PREFIX, `Cannot inject script into tab ${tabId}:`, err) + } + } +} + +// ─── Tab Lifecycle Monitoring ────────────────────────────────────── + +function onTabCreated(tab: chrome.tabs.Tab) { + if (!isRecording) return + const seq = recordingSeqId + + if (tab.id) { + // Start recording in new tab once it's ready + ensureContentScriptInTab(tab.id) + + const event: RawRecordingEvent = { + type: 'newTab', + timestamp: Date.now(), + url: tab.url || '', + title: tab.title || '', + tabId: tab.id, + data: { url: tab.url || '' }, + } + forwardToSidepanel(event) + } + + refreshTabIndexMap().then(() => { + if (recordingSeqId !== seq) return // stale + }) +} + +function onTabActivated(activeInfo: { tabId: number; windowId: number }) { + if (!isRecording) return + const seq = recordingSeqId + + refreshTabIndexMap().then(() => { + if (recordingSeqId !== seq) return // stale + const tabIdx = getTabIdx(activeInfo.tabId) + chrome.tabs + .get(activeInfo.tabId) + .then((tab) => { + if (recordingSeqId !== seq) return // stale + const event: RawRecordingEvent = { + type: 'switchTab', + timestamp: Date.now(), + url: tab.url || '', + title: tab.title || '', + tabId: activeInfo.tabId, + data: { tabIdx }, + } + forwardToSidepanel(event) + }) + .catch(() => { + // Tab may have been closed already + }) + }) +} + +function onTabRemoved(tabId: number) { + if (!isRecording) return + const tabIdx = getTabIdx(tabId) + lastNavigateUrl.delete(tabId) + + const event: RawRecordingEvent = { + type: 'closeTab', + timestamp: Date.now(), + url: '', + title: '', + tabId, + data: { tabIdx }, + } + forwardToSidepanel(event) + + refreshTabIndexMap() +} + +function onTabUpdated(tabId: number, changeInfo: { url?: string; status?: string }, tab: chrome.tabs.Tab) { + if (!isRecording) return + + // Track URL navigation — dedup same-URL events + if (changeInfo.url) { + const lastUrl = lastNavigateUrl.get(tabId) + if (lastUrl === changeInfo.url) return // skip duplicate + lastNavigateUrl.set(tabId, changeInfo.url) + + const event: RawRecordingEvent = { + type: 'navigate', + timestamp: Date.now(), + url: changeInfo.url, + title: tab.title || '', + tabId, + data: { url: changeInfo.url }, + } + forwardToSidepanel(event) + } + + // When a tab finishes loading, make sure content script is recording + if (changeInfo.status === 'complete') { + chrome.tabs + .sendMessage(tabId, { + type: 'RECORDING_CONTROL', + action: 'status', + payload: { isRecording: true }, + }) + .catch(() => { + // Content script not yet loaded, will pick it up via query_status + }) + } +} + +// ─── Tab Listeners ───────────────────────────────────────────────── + +let tabListenersRegistered = false + +function registerTabListeners() { + if (tabListenersRegistered) return + tabListenersRegistered = true + + chrome.tabs.onCreated.addListener(onTabCreated) + chrome.tabs.onActivated.addListener(onTabActivated) + chrome.tabs.onRemoved.addListener(onTabRemoved) + chrome.tabs.onUpdated.addListener(onTabUpdated) +} + +function unregisterTabListeners() { + if (!tabListenersRegistered) return + tabListenersRegistered = false + + chrome.tabs.onCreated.removeListener(onTabCreated) + chrome.tabs.onActivated.removeListener(onTabActivated) + chrome.tabs.onRemoved.removeListener(onTabRemoved) + chrome.tabs.onUpdated.removeListener(onTabUpdated) +} + +// ─── Recording Control ──────────────────────────────────────────── + +async function startRecording() { + recordingSeqId++ + isRecording = true + recordingStartTime = Date.now() + lastNavigateUrl.clear() + await refreshTabIndexMap() + registerTabListeners() + + // Persist state for service worker restart recovery + try { + await chrome.storage.session.set({ + isRecordingActive: true, + recordingStartTime, + }) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Failed to persist recording state:', err) + } + + // Broadcast to all content scripts + await broadcastToContentScripts({ + type: 'RECORDING_CONTROL', + action: 'start', + }) + + console.debug(DEBUG_PREFIX, 'Recording started') +} + +async function stopRecording() { + recordingSeqId++ + isRecording = false + recordingStartTime = 0 + lastNavigateUrl.clear() + unregisterTabListeners() + + try { + await chrome.storage.session.set({ + isRecordingActive: false, + recordingStartTime: 0, + }) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Failed to clear recording state:', err) + } + + // Broadcast to all content scripts + await broadcastToContentScripts({ + type: 'RECORDING_CONTROL', + action: 'stop', + }) + + console.debug(DEBUG_PREFIX, 'Recording stopped') +} + +// ─── Message Handler ─────────────────────────────────────────────── + +export function handleRecordingControlMessage( + message: any, + sender: chrome.runtime.MessageSender, + sendResponse: (response: unknown) => void +): true | undefined { + if (message.type !== 'RECORDING_CONTROL') return + + switch (message.action) { + case 'start': { + startRecording() + .then(() => sendResponse({ success: true })) + .catch((err) => sendResponse({ success: false, error: String(err) })) + return true + } + + case 'stop': { + stopRecording() + .then(() => sendResponse({ success: true })) + .catch((err) => sendResponse({ success: false, error: String(err) })) + return true + } + + case 'query_status': { + sendResponse({ isRecording }) + return + } + + case 'recording_event': { + // Only handle events from content scripts (sender.tab exists) + // Ignore messages from ourselves (forwardToSidepanel re-entry) + if (!sender.tab) return + + if (!isRecording) { + sendResponse({ success: false, error: 'Not recording' }) + return + } + + try { + const payload = message.payload as Omit + const tabId = sender.tab.id ?? 0 + + const event: RawRecordingEvent = { + ...payload, + tabId, + } + + forwardToSidepanel(event) + sendResponse({ success: true }) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Error processing recording event:', err) + sendResponse({ success: false, error: String(err) }) + } + return + } + + default: + return + } +} + +// ─── Service Worker Restart Recovery ─────────────────────────────── + +/** Max age for a recording session to be recovered (30 minutes) */ +const MAX_RECOVERY_AGE_MS = 30 * 60 * 1000 + +export async function recoverRecordingState() { + try { + const result = await chrome.storage.session.get(['isRecordingActive', 'recordingStartTime']) + if (!result.isRecordingActive) return + + // Validate the recording isn't too old (stale state from a crash) + const startTime = result.recordingStartTime as number | undefined + if (startTime && Date.now() - startTime > MAX_RECOVERY_AGE_MS) { + console.debug(DEBUG_PREFIX, 'Recording session too old, clearing stale state') + await chrome.storage.session.set({ isRecordingActive: false, recordingStartTime: 0 }) + return + } + + console.debug(DEBUG_PREFIX, 'Recovering recording state after SW restart') + isRecording = true + recordingStartTime = startTime || Date.now() + registerTabListeners() + await refreshTabIndexMap() + + // Re-notify all content scripts + await broadcastToContentScripts({ + type: 'RECORDING_CONTROL', + action: 'status', + payload: { isRecording: true }, + }) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Recovery error:', err) + } +} diff --git a/packages/extension/src/agent/EventRecorder.content.ts b/packages/extension/src/agent/EventRecorder.content.ts new file mode 100644 index 000000000..1133eb890 --- /dev/null +++ b/packages/extension/src/agent/EventRecorder.content.ts @@ -0,0 +1,600 @@ +/** + * EventRecorder — Content Script side + * + * Listens to user interactions (click, input, change, keydown, scroll) + * and reports them as RawRecordingEvents to the background script. + * + * Handles edge cases: iframe, shadow DOM, contenteditable, file upload. + * Captures highlightIndex from PageController for element identification. + */ + +import { PageController } from '@page-agent/page-controller' +import type { ElementDescriptor, RawRecordingEvent } from '@/lib/recording-types' + +const DEBUG_PREFIX = '[EventRecorder.content]' + +let isRecording = false +let scrollTimer: ReturnType | null = null +let lastScrollY = 0 +let inputDebounceTimers = new Map>() + +// ─── Shared PageController for element indexing ───────────────────── + +let recordingPageController: PageController | null = null +let indexRefreshTimer: ReturnType | null = null + +function getRecordingPageController(): PageController { + if (!recordingPageController) { + recordingPageController = new PageController({ enableMask: false, viewportExpansion: -1 }) + } + return recordingPageController +} + +/** + * Update the DOM tree and cache highlightIndex into element dataset. + * Called on recording start and periodically during recording. + */ +async function refreshElementIndices() { + try { + const pc = getRecordingPageController() + await pc.updateTree() + + const selectorMap = (pc as any).selectorMap as Map + selectorMap.forEach((node: any, index: number) => { + if (node.ref instanceof HTMLElement) { + node.ref.dataset.pageAgentIdx = String(index) + } + }) + + console.debug(DEBUG_PREFIX, `Indexed ${selectorMap.size} interactive elements`) + } catch { + // Ignore errors — page may not be ready + } +} + +/** + * Schedule periodic index refresh during recording. + * Re-indexes every 3 seconds to catch dynamic DOM changes. + */ +function startIndexRefresh() { + stopIndexRefresh() + // Initial index + refreshElementIndices() + indexRefreshTimer = setInterval(() => { + refreshElementIndices() + }, 3000) +} + +function stopIndexRefresh() { + if (indexRefreshTimer) { + clearInterval(indexRefreshTimer) + indexRefreshTimer = null + } +} + +/** + * Get the highlightIndex for a specific element. + * First checks cache, then does a targeted lookup. + */ +async function getElementHighlightIndex(element: Element): Promise { + // Check cache first + if (element instanceof HTMLElement && element.dataset.pageAgentIdx) { + return parseInt(element.dataset.pageAgentIdx, 10) + } + + try { + const pc = getRecordingPageController() + await pc.updateTree() + + const selectorMap = (pc as any).selectorMap as Map + for (const [index, node] of selectorMap.entries()) { + if (node.ref === element) { + if (element instanceof HTMLElement) { + element.dataset.pageAgentIdx = String(index) + } + return index + } + } + return undefined + } catch { + return undefined + } +} + +// ─── Element Descriptor Builder ──────────────────────────────────── + +function buildElementDescriptor(el: Element): ElementDescriptor { + const tag = el.tagName.toLowerCase() + + // Get highlightIndex from dataset cache + let idx: number | undefined = undefined + if (el instanceof HTMLElement && el.dataset.pageAgentIdx) { + idx = parseInt(el.dataset.pageAgentIdx, 10) + } + + // If not cached, trigger async refresh (result will be available for next event) + if (idx === undefined && isRecording) { + getElementHighlightIndex(el) + } + + // Visible text — trim and limit + let text = '' + if (el instanceof HTMLElement) { + if (el.isContentEditable) { + text = (el.textContent || '').trim().slice(0, 100) + } else { + text = (el.innerText || el.textContent || '').trim().slice(0, 100) + } + } + + const role = el.getAttribute('role') || undefined + const ariaLabel = el.getAttribute('aria-label') || undefined + const placeholder = el.getAttribute('placeholder') || undefined + const name = el.getAttribute('name') || undefined + + // Best-effort CSS selector + const selector = buildSelector(el) + + // Context — nearest landmark or heading + const context = findContext(el) + + // For file inputs, add type info + const extras: Partial = {} + if (tag === 'input') { + const inputType = el.getAttribute('type') + if (inputType === 'file') { + extras.role = 'file-upload' + } + } + + // For contenteditable, mark it + if (el instanceof HTMLElement && el.isContentEditable && tag !== 'input' && tag !== 'textarea') { + extras.role = extras.role || 'textbox' + } + + return { + text, + tag, + ...(role && { role }), + ...(ariaLabel && { ariaLabel }), + ...(placeholder && { placeholder }), + ...(name && { name }), + ...(selector && { selector }), + ...(context && { context }), + ...(idx !== undefined && { idx }), + ...extras, + } +} + +function buildSelector(el: Element): string | undefined { + try { + // If element is inside a shadow root, prefix with host selector + const shadowPrefix = getShadowHostSelector(el) + + // Try id + if (el.id) { + const sel = `#${CSS.escape(el.id)}` + return shadowPrefix ? `${shadowPrefix} >>> ${sel}` : sel + } + + // Try unique class combination + const tag = el.tagName.toLowerCase() + const classes = Array.from(el.classList) + .filter((c) => !c.startsWith('__') && !c.startsWith('css-') && c.length < 40) + .slice(0, 3) + + if (classes.length > 0) { + const sel = `${tag}.${classes.map((c) => CSS.escape(c)).join('.')}` + const root = el.getRootNode() + const doc = root instanceof Document ? root : (root as ShadowRoot) + try { + if (doc.querySelectorAll(sel).length === 1) { + return shadowPrefix ? `${shadowPrefix} >>> ${sel}` : sel + } + } catch { + // Invalid selector, skip + } + } + + // Try tag + attributes + const attrs = ['name', 'type', 'role', 'aria-label', 'placeholder', 'data-testid'] + for (const attr of attrs) { + const val = el.getAttribute(attr) + if (val) { + const sel = `${tag}[${attr}=${JSON.stringify(val)}]` + const root = el.getRootNode() + const doc = root instanceof Document ? root : (root as ShadowRoot) + try { + if (doc.querySelectorAll(sel).length === 1) { + return shadowPrefix ? `${shadowPrefix} >>> ${sel}` : sel + } + } catch { + // Invalid selector, skip + } + } + } + + return undefined + } catch { + return undefined + } +} + +/** Get a selector path for the shadow DOM host element, if any */ +function getShadowHostSelector(el: Element): string | undefined { + try { + const root = el.getRootNode() + if (root instanceof ShadowRoot && root.host) { + const host = root.host + if (host.id) return `#${CSS.escape(host.id)}` + const tag = host.tagName.toLowerCase() + // Custom elements usually have unique tag names + if (tag.includes('-')) return tag + return undefined + } + return undefined + } catch { + return undefined + } +} + +function findContext(el: Element): string | undefined { + const landmarks = [ + 'header', + 'nav', + 'main', + 'aside', + 'footer', + '[role="banner"]', + '[role="navigation"]', + '[role="main"]', + '[role="search"]', + '[role="dialog"]', + '[role="alertdialog"]', + '[role="complementary"]', + '[role="contentinfo"]', + '[role="form"]', + '[role="region"]', + ] + + let current: Element | null = el.parentElement + const maxDepth = 15 + let depth = 0 + + while (current && depth < maxDepth) { + // Check if it's a landmark + const tagLower = current.tagName.toLowerCase() + const role = current.getAttribute('role') + + if (landmarks.includes(tagLower) || (role && landmarks.includes(`[role="${role}"]`))) { + const label = current.getAttribute('aria-label') || tagLower + return label + } + + // Check for heading + if (/^h[1-6]$/.test(tagLower)) { + return (current as HTMLElement).innerText?.trim().slice(0, 50) || tagLower + } + + // Cross shadow DOM boundary — walk to host + if (!current.parentElement) { + const root = current.getRootNode() + if (root instanceof ShadowRoot && root.host) { + current = root.host + continue + } + } + + current = current.parentElement + depth++ + } + + return undefined +} + +// ─── Resolve Target from Event ───────────────────────────────────── + +/** Resolve the actual target element, crossing shadow DOM if needed */ +function resolveTarget(e: Event): Element | null { + // composedPath()[0] gives the actual target, even across shadow boundaries + const path = e.composedPath() + if (path.length > 0 && path[0] instanceof Element) { + return path[0] + } + return e.target instanceof Element ? e.target : null +} + +// ─── Event Sending ───────────────────────────────────────────────── + +function sendEvent(event: Omit) { + if (!isRecording) return + + try { + const rawEvent: Omit = { + ...event, + timestamp: Date.now(), + url: window.location.href, + title: document.title, + } + + chrome.runtime + .sendMessage({ + type: 'RECORDING_CONTROL', + action: 'recording_event', + payload: rawEvent, + }) + .catch((err) => { + console.debug(DEBUG_PREFIX, 'Failed to send event:', err) + }) + } catch (err) { + console.debug(DEBUG_PREFIX, 'Error building event:', err) + } +} + +// ─── Event Handlers ──────────────────────────────────────────────── + +function handleClick(e: MouseEvent) { + if (!isRecording) return + const target = resolveTarget(e) + if (!target) return + + // For file inputs, record as a special click + if ( + target instanceof HTMLInputElement && + target.type === 'file' + ) { + sendEvent({ + type: 'click', + el: buildElementDescriptor(target), + data: { fileInput: true }, + }) + return + } + + sendEvent({ + type: 'click', + el: buildElementDescriptor(target), + }) +} + +function handleInput(e: Event) { + if (!isRecording) return + const target = resolveTarget(e) + if (!target) return + + // Handle contenteditable elements + if (target instanceof HTMLElement && target.isContentEditable) { + const existingTimer = inputDebounceTimers.get(target) + if (existingTimer) clearTimeout(existingTimer) + + inputDebounceTimers.set( + target, + setTimeout(() => { + inputDebounceTimers.delete(target) + sendEvent({ + type: 'input', + el: buildElementDescriptor(target), + data: { value: target.textContent || '' }, + }) + }, 500) + ) + return + } + + // Standard input/textarea + if (!('value' in target)) return + const inputTarget = target as HTMLInputElement | HTMLTextAreaElement + + // Skip file inputs — their value changes are not user-typed text + if (inputTarget instanceof HTMLInputElement && inputTarget.type === 'file') return + + // Debounce input events — wait 500ms of inactivity + const existingTimer = inputDebounceTimers.get(inputTarget) + if (existingTimer) clearTimeout(existingTimer) + + inputDebounceTimers.set( + inputTarget, + setTimeout(() => { + inputDebounceTimers.delete(inputTarget) + sendEvent({ + type: 'input', + el: buildElementDescriptor(inputTarget), + data: { value: inputTarget.value }, + }) + }, 500) + ) +} + +function handleChange(e: Event) { + if (!isRecording) return + const target = resolveTarget(e) + if (!target) return + + // File upload — record the selected file names + if (target instanceof HTMLInputElement && target.type === 'file') { + const files = target.files + if (files && files.length > 0) { + const fileNames = Array.from(files).map((f) => f.name) + sendEvent({ + type: 'input', + el: buildElementDescriptor(target), + data: { value: fileNames.join(', '), fileUpload: true, fileNames }, + }) + } + return + } + + // Select element + if (target.tagName.toLowerCase() !== 'select') return + const selectTarget = target as HTMLSelectElement + + sendEvent({ + type: 'select', + el: buildElementDescriptor(selectTarget), + data: { value: selectTarget.value }, + }) +} + +function handleKeyDown(e: KeyboardEvent) { + if (!isRecording) return + + // Only capture special keys, not regular typing + const specialKeys = [ + 'Enter', + 'Escape', + 'Tab', + 'Backspace', + 'Delete', + 'ArrowUp', + 'ArrowDown', + 'ArrowLeft', + 'ArrowRight', + 'Home', + 'End', + 'PageUp', + 'PageDown', + ] + const hasModifier = e.ctrlKey || e.metaKey || e.altKey + + if (!specialKeys.includes(e.key) && !hasModifier) return + + const modifiers: string[] = [] + if (e.ctrlKey) modifiers.push('Ctrl') + if (e.metaKey) modifiers.push('Meta') + if (e.altKey) modifiers.push('Alt') + if (e.shiftKey) modifiers.push('Shift') + + const target = resolveTarget(e) + + sendEvent({ + type: 'keypress', + el: target ? buildElementDescriptor(target) : undefined, + data: { + key: e.key, + ...(modifiers.length > 0 && { modifiers }), + }, + }) +} + +function handleScroll() { + if (!isRecording) return + + // Debounce scroll — aggregate into single event after 300ms of inactivity + if (scrollTimer) clearTimeout(scrollTimer) + + scrollTimer = setTimeout(() => { + const currentY = window.scrollY + const delta = currentY - lastScrollY + if (Math.abs(delta) < 50) return // ignore tiny scrolls + + sendEvent({ + type: 'scroll', + data: { + direction: delta > 0 ? 'down' : 'up', + pixels: Math.abs(Math.round(delta)), + }, + }) + + lastScrollY = currentY + scrollTimer = null + }, 300) +} + +// ─── Lifecycle ───────────────────────────────────────────────────── + +function startRecording() { + if (isRecording) return + isRecording = true + lastScrollY = window.scrollY + + // Start periodic element indexing + startIndexRefresh() + + document.addEventListener('click', handleClick, { capture: true }) + document.addEventListener('input', handleInput, { capture: true }) + document.addEventListener('change', handleChange, { capture: true }) + document.addEventListener('keydown', handleKeyDown, { capture: true }) + window.addEventListener('scroll', handleScroll, { passive: true }) + + console.debug(DEBUG_PREFIX, 'Recording started') +} + +function stopRecording() { + if (!isRecording) return + isRecording = false + + // Stop periodic element indexing + stopIndexRefresh() + + document.removeEventListener('click', handleClick, { capture: true }) + document.removeEventListener('input', handleInput, { capture: true }) + document.removeEventListener('change', handleChange, { capture: true }) + document.removeEventListener('keydown', handleKeyDown, { capture: true }) + window.removeEventListener('scroll', handleScroll) + + // Clean up PageController + if (recordingPageController) { + recordingPageController.dispose() + recordingPageController = null + } + + // Clear pending timers + if (scrollTimer) { + clearTimeout(scrollTimer) + scrollTimer = null + } + for (const timer of inputDebounceTimers.values()) { + clearTimeout(timer) + } + inputDebounceTimers.clear() + + console.debug(DEBUG_PREFIX, 'Recording stopped') +} + +// ─── Init ────────────────────────────────────────────────────────── + +export function initEventRecorder() { + chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + if (message.type !== 'RECORDING_CONTROL') return + + switch (message.action) { + case 'start': + startRecording() + sendResponse({ success: true }) + break + case 'stop': + stopRecording() + sendResponse({ success: true }) + break + case 'status': + // Background is informing us about recording state + if (message.payload?.isRecording) { + startRecording() + } else { + stopRecording() + } + sendResponse({ success: true }) + break + default: + // Ignore other recording control messages + break + } + }) + + // On load, check if a recording is in progress + chrome.runtime + .sendMessage({ + type: 'RECORDING_CONTROL', + action: 'query_status', + }) + .then((response: any) => { + if (response?.isRecording) { + startRecording() + } + }) + .catch(() => { + // Background not ready yet, ignore + }) + + console.debug(DEBUG_PREFIX, 'Event recorder initialized') +} diff --git a/packages/extension/src/agent/RecordingReplayAgent.ts b/packages/extension/src/agent/RecordingReplayAgent.ts new file mode 100644 index 000000000..39f2976be --- /dev/null +++ b/packages/extension/src/agent/RecordingReplayAgent.ts @@ -0,0 +1,189 @@ +/** + * RecordingReplayAgent — Converts a Recording into task + systemInstruction + * for the existing PageAgentCore execution loop. + * + * Core idea: the recording becomes a "plan" injected via systemInstruction. + * The LLM reads the plan + observes the actual page → adaptive execution. + * Zero changes to PageAgentCore. + */ + +import type { Recording, RecordedStep, ElementDescriptor } from '@/lib/recording-types' +import REPLAY_PROMPT_TEMPLATE from './replay_prompt.md?raw' + +// ─── Step Formatting ─────────────────────────────────────────────── + +/** Format a single step into a compact one-line string for LLM consumption (~10 tokens/step) */ +function formatStepForLLM(step: RecordedStep, index: number): string { + const prefix = `[${index + 1}]` + const act = step.act + const el = step.el + + switch (act.type) { + case 'click': { + const target = describeElement(el) + const context = el?.context ? ` in ${el.context}` : '' + const idxNote = el?.idx !== undefined ? ` (index:${el.idx})` : '' + return `${prefix} click ${target}${idxNote}${context}` + } + + case 'input': { + const target = describeElement(el) + const paramNote = act.param ? ` [PARAM:${act.param}]` : '' + const idxNote = el?.idx !== undefined ? ` (index:${el.idx})` : '' + return `${prefix} type "${act.value}" → ${target}${idxNote}${paramNote}` + } + + case 'select': { + const target = describeElement(el) + const paramNote = act.param ? ` [PARAM:${act.param}]` : '' + const idxNote = el?.idx !== undefined ? ` (index:${el.idx})` : '' + return `${prefix} select "${act.value}" in ${target}${idxNote}${paramNote}` + } + + case 'scroll': + return `${prefix} scroll ${act.direction} ${act.pixels}px` + + case 'navigate': { + const paramNote = act.param ? ` [PARAM:${act.param}]` : '' + return `${prefix} navigate to ${act.url}${paramNote}` + } + + case 'newTab': + return `${prefix} open new tab: ${act.url}` + + case 'switchTab': + return `${prefix} switch to tab ${act.tabIdx}` + + case 'closeTab': + return `${prefix} close tab ${act.tabIdx}` + + case 'keypress': { + const mods = act.modifiers?.join('+') ?? '' + const key = mods ? `${mods}+${act.key}` : act.key + const target = el ? ` → ${describeElement(el)}` : '' + const idxNote = el?.idx !== undefined ? ` (index:${el.idx})` : '' + return `${prefix} press ${key}${target}${idxNote}` + } + + case 'wait': + return `${prefix} wait ${act.seconds}s` + + default: + return `${prefix} ${JSON.stringify(act)}` + } +} + +/** Create a compact element description string */ +function describeElement(el?: ElementDescriptor): string { + if (!el) return 'element' + + const parts: string[] = [] + + // Text is highest priority + if (el.text) { + parts.push(`"${el.text.slice(0, 40)}"`) + } + + // Tag + role + if (el.tag) { + const roleStr = el.role ? ` role="${el.role}"` : '' + parts.push(`(${el.tag}${roleStr})`) + } + + // Additional attributes for disambiguation + if (el.ariaLabel) parts.push(`aria-label="${el.ariaLabel}"`) + if (el.placeholder) parts.push(`placeholder="${el.placeholder}"`) + if (el.name) parts.push(`name="${el.name}"`) + + // URL context + if (parts.length === 0 && el.selector) { + parts.push(el.selector) + } + + return parts.join(' ') || 'element' +} + +// ─── Parameter Handling ──────────────────────────────────────────── + +/** Extract all param names from a recording */ +export function extractParams(recording: Recording): Map { + const params = new Map() + + for (const step of recording.steps) { + const act = step.act + if ('param' in act && act.param && 'value' in act) { + params.set(act.param, act.value as string) + } + } + + return params +} + +/** Apply parameter overrides to a recording (returns a new copy) */ +export function applyParamOverrides( + recording: Recording, + overrides: Record +): Recording { + const newSteps = recording.steps.map((step) => { + const act = step.act + if ('param' in act && act.param && act.param in overrides) { + return { + ...step, + act: { ...act, value: overrides[act.param] }, + } + } + return step + }) + + return { ...recording, steps: newSteps } +} + +// ─── Task Building ───────────────────────────────────────────────── + +export interface ReplayTaskResult { + task: string + systemInstruction: string +} + +/** + * Build a replay task + system instruction from a Recording. + * + * @param recording - The recording to replay + * @param paramOverrides - Optional parameter value overrides + * @param nlModification - Optional natural language modification instruction + */ +export function buildReplayTask( + recording: Recording, + paramOverrides?: Record, + nlModification?: string +): ReplayTaskResult { + // Apply param overrides + const effectiveRecording = paramOverrides + ? applyParamOverrides(recording, paramOverrides) + : recording + + // Format steps as compact plan + const planLines = effectiveRecording.steps.map((step, i) => formatStepForLLM(step, i)) + const plan = planLines.join('\n') + + // Build system instruction from template + let systemInstruction = REPLAY_PROMPT_TEMPLATE.replace('{{PLAN}}', plan) + + // Handle natural language modification + if (nlModification?.trim()) { + systemInstruction = systemInstruction.replace('{{NL_MOD}}', nlModification.trim()) + } else { + // Remove the NL modification section entirely (handle any line endings) + systemInstruction = systemInstruction.replace( + /\s*\{\{NL_MOD\}\}\s*<\/natural_language_modification>/, + '' + ) + } + + // Build task description + const taskDesc = recording.desc || recording.name || 'Replay recorded browser automation' + const startUrlNote = recording.startUrl ? `\nStart at: ${recording.startUrl}` : '' + const task = `${taskDesc}${startUrlNote}\n\nFollow the plan in the system instruction to complete this task.` + + return { task, systemInstruction } +} diff --git a/packages/extension/src/agent/autoNameRecording.ts b/packages/extension/src/agent/autoNameRecording.ts new file mode 100644 index 000000000..90ea63dd4 --- /dev/null +++ b/packages/extension/src/agent/autoNameRecording.ts @@ -0,0 +1,117 @@ +/** + * autoNameRecording — Call LLM to generate name and description for a recording + * + * Uses a lightweight direct API call (no tool-calling overhead). + * Falls back to empty strings if the call fails. + */ + +import type { LLMConfig } from '@page-agent/llms' +import type { RecordedStep, Recording } from '@/lib/recording-types' + +/** Compact step summary for the naming prompt (~5 tokens per step) */ +function summarizeStep(step: RecordedStep, index: number): string { + const act = step.act + const el = step.el + const elDesc = el?.text ? `"${el.text.slice(0, 30)}"` : el?.tag || '' + + switch (act.type) { + case 'click': + return `${index + 1}. click ${elDesc}` + case 'input': + return `${index + 1}. type "${act.value.slice(0, 20)}" → ${elDesc}` + case 'select': + return `${index + 1}. select "${act.value}" in ${elDesc}` + case 'scroll': + return `${index + 1}. scroll ${act.direction}` + case 'navigate': + return `${index + 1}. go to ${act.url}` + case 'newTab': + return `${index + 1}. new tab: ${act.url}` + case 'switchTab': + return `${index + 1}. switch tab` + case 'closeTab': + return `${index + 1}. close tab` + case 'keypress': + return `${index + 1}. press ${act.key}` + case 'wait': + return `${index + 1}. wait ${act.seconds}s` + default: + return `${index + 1}. ${act.type}` + } +} + +const NAMING_PROMPT = `You are a concise naming assistant. Given a sequence of browser actions, generate: +1. A short name (2-6 words, like "搜索B站视频" or "Login to GitHub") +2. A one-sentence description of what the action sequence does + +Respond in JSON format: {"name": "...", "desc": "..."} +Use the same language as the page titles and content. Do NOT include any markdown or explanation.` + +export interface AutoNameResult { + name: string + desc: string +} + +/** + * Call LLM to generate a name and description for the recording. + * Returns { name: '', desc: '' } on failure. + */ +export async function autoNameRecording( + steps: RecordedStep[], + startUrl: string, + llmConfig: LLMConfig +): Promise { + const fallback: AutoNameResult = { name: '', desc: '' } + + if (!llmConfig?.baseURL || !llmConfig?.apiKey || !llmConfig?.model) { + return fallback + } + + if (steps.length === 0) { + return fallback + } + + // Build compact step summary + const stepsSummary = steps.map((s, i) => summarizeStep(s, i)).join('\n') + const userMsg = `Start URL: ${startUrl}\n\nActions:\n${stepsSummary}` + + try { + const response = await fetch(`${llmConfig.baseURL}/chat/completions`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${llmConfig.apiKey}`, + }, + body: JSON.stringify({ + model: llmConfig.model, + temperature: 0.3, + messages: [ + { role: 'system', content: NAMING_PROMPT }, + { role: 'user', content: userMsg }, + ], + }), + }) + + if (!response.ok) { + console.debug('[autoNameRecording] API error:', response.status) + return fallback + } + + const data = await response.json() + const content = data?.choices?.[0]?.message?.content?.trim() + + if (!content) return fallback + + // Parse JSON — handle possible markdown wrapping + const jsonStr = content.replace(/^```json?\s*/i, '').replace(/\s*```$/, '') + const parsed = JSON.parse(jsonStr) + + return { + name: typeof parsed.name === 'string' ? parsed.name.slice(0, 50) : '', + desc: typeof parsed.desc === 'string' ? parsed.desc.slice(0, 200) : '', + } + } catch (err) { + console.debug('[autoNameRecording] Failed:', err) + return fallback + } +} diff --git a/packages/extension/src/agent/replay_prompt.md b/packages/extension/src/agent/replay_prompt.md new file mode 100644 index 000000000..ce3038146 --- /dev/null +++ b/packages/extension/src/agent/replay_prompt.md @@ -0,0 +1,31 @@ +You are replaying a pre-recorded browser automation plan. Your goal is to follow the plan steps while adapting to the actual page state. + + +{{PLAN}} + + + +1. **Follow the plan step by step** — execute each step in order. +2. **Element matching with index** — when a step includes `(index:N)`, prefer using that index directly with `click_element_by_index(N)` or `input_text(index=N, ...)`. The index corresponds to the highlighted interactive element number on the page. +3. **Semantic fallback** — if the index doesn't match the expected element (wrong text/role), fall back to semantic matching using multiple signals: + - Visible text content (highest priority) + - aria-label attribute + - ARIA role + - placeholder text + - Element tag name + - Nearby landmark/heading context + - CSS selector (lowest priority, for fallback) +4. **Parameter substitution** — values marked with [PARAM:name] have been replaced with user-provided values. Use the substituted values. +5. **Adaptive execution** — if a step fails or the page looks different from expected: + - Try the index first, then alternative approaches (e.g., different element matching) + - If a navigation changed the URL pattern, adapt accordingly + - Skip steps that are no longer relevant (e.g., element already visible) + - Never get stuck — if you can't find an element after reasonable attempts, move to the next step +6. **Wait for page loads** — after navigation or clicks that trigger page changes, wait for the page to stabilize before proceeding. +7. **Report progress** — in your memory, track which plan step you're on (e.g., "Completed step 3/5, now on step 4"). +8. **Completion** — after executing all steps (or determining remaining steps are impossible), call `done` with appropriate success/failure status. + + + +{{NL_MOD}} + diff --git a/packages/extension/src/agent/useRecording.ts b/packages/extension/src/agent/useRecording.ts new file mode 100644 index 000000000..2a5ba4e19 --- /dev/null +++ b/packages/extension/src/agent/useRecording.ts @@ -0,0 +1,297 @@ +/** + * useRecording — React Hook for recording management + * + * Manages recording state (idle/recording), start/stop, + * real-time event stream, raw event → RecordedStep conversion, + * and saving to IndexedDB. + * + * Phase 6: error handling, defensive coding + */ +import type { LLMConfig } from '@page-agent/llms' +import { useCallback, useEffect, useRef, useState } from 'react' + +import { autoNameRecording } from '@/agent/autoNameRecording' +import { saveRecording, updateRecording } from '@/lib/db' +import type { RawRecordingEvent, RecordedStep, Recording } from '@/lib/recording-types' + +export type RecordingState = 'idle' | 'recording' + +export interface UseRecordingResult { + recordingState: RecordingState + steps: RecordedStep[] + startRecording: () => Promise + stopRecording: () => Promise + discardRecording: () => void + eventCount: number + error: string | null +} + +/** Convert a RawRecordingEvent to a RecordedStep */ +function rawToStep( + raw: RawRecordingEvent, + startTime: number, + tabIdxMap: Map +): RecordedStep | null { + try { + const dt = raw.timestamp - startTime + const tabIdx = tabIdxMap.get(raw.tabId) ?? 0 + const page = { url: raw.url, title: raw.title, tabIdx } + + switch (raw.type) { + case 'click': + return { act: { type: 'click' }, page, el: raw.el, dt } + + case 'input': + return { + act: { type: 'input', value: String(raw.data?.value ?? '') }, + page, + el: raw.el, + dt, + } + + case 'select': + return { + act: { type: 'select', value: String(raw.data?.value ?? '') }, + page, + el: raw.el, + dt, + } + + case 'scroll': + return { + act: { + type: 'scroll', + direction: (raw.data?.direction as 'up' | 'down') ?? 'down', + pixels: Number(raw.data?.pixels ?? 300), + }, + page, + dt, + } + + case 'keypress': { + const modifiers = raw.data?.modifiers as string[] | undefined + return { + act: { + type: 'keypress', + key: String(raw.data?.key ?? ''), + ...(modifiers ? { modifiers } : {}), + }, + page, + el: raw.el, + dt, + } + } + + case 'navigate': + return { + act: { type: 'navigate', url: String(raw.data?.url ?? raw.url) }, + page, + dt, + } + + case 'newTab': + return { + act: { type: 'newTab', url: String(raw.data?.url ?? raw.url) }, + page, + dt, + } + + case 'switchTab': + return { + act: { type: 'switchTab', tabIdx: Number(raw.data?.tabIdx ?? tabIdx) }, + page, + dt, + } + + case 'closeTab': + return { + act: { type: 'closeTab', tabIdx: Number(raw.data?.tabIdx ?? tabIdx) }, + page, + dt, + } + + default: + return null + } + } catch (err) { + console.debug('[useRecording] Failed to convert raw event:', err) + return null + } +} + +export function useRecording(): UseRecordingResult { + const [recordingState, setRecordingState] = useState('idle') + const [steps, setSteps] = useState([]) + const [error, setError] = useState(null) + const startTimeRef = useRef(0) + const startUrlRef = useRef('') + const tabIdxMapRef = useRef(new Map()) + const rawEventsRef = useRef([]) + const stepsRef = useRef([]) + const recordingStateRef = useRef('idle') + + // Keep refs in sync with state + useEffect(() => { + stepsRef.current = steps + }, [steps]) + useEffect(() => { + recordingStateRef.current = recordingState + }, [recordingState]) + // Listen for recording events from background + useEffect(() => { + const listener = (message: any) => { + if ( + message.type !== 'RECORDING_CONTROL' || + message.action !== 'recording_event' + ) { + return + } + + // Guard: ignore events when not recording + if (recordingStateRef.current !== 'recording') return + + try { + const raw = message.payload as RawRecordingEvent + + // Track tab indices + if (!tabIdxMapRef.current.has(raw.tabId)) { + tabIdxMapRef.current.set(raw.tabId, tabIdxMapRef.current.size) + } + + rawEventsRef.current.push(raw) + + const step = rawToStep(raw, startTimeRef.current, tabIdxMapRef.current) + if (step) { + setSteps((prev) => [...prev, step]) + } + } catch (err) { + console.debug('[useRecording] Error processing event:', err) + } + } + + chrome.runtime.onMessage.addListener(listener) + return () => chrome.runtime.onMessage.removeListener(listener) + }, []) + + // ─── Auto-naming helper ───────────────────────────────────────────── + /** Call LLM to generate name/desc in background, update DB silently */ + async function autoNameInBackground(recording: Recording) { + try { + const result = await chrome.storage.local.get('llmConfig') + const llmConfig = result.llmConfig as LLMConfig | undefined + if (!llmConfig) return + + const { name, desc } = await autoNameRecording( + recording.steps, + recording.startUrl, + llmConfig + ) + + if (name || desc) { + const updated = { + ...recording, + name: name || recording.name, + desc: desc || recording.desc, + } + await updateRecording(updated) + console.debug('[useRecording] Auto-named recording:', name) + } + } catch (err) { + console.debug('[useRecording] Auto-naming failed (non-critical):', err) + } + } + + const startRecording = useCallback(async () => { + setError(null) + + // Get current tab info for startUrl + try { + const [tab] = await chrome.tabs.query({ active: true, currentWindow: true }) + startUrlRef.current = tab?.url || '' + } catch { + startUrlRef.current = '' + } + + startTimeRef.current = Date.now() + rawEventsRef.current = [] + tabIdxMapRef.current.clear() + setSteps([]) + setRecordingState('recording') + + // Tell background to start + try { + await chrome.runtime.sendMessage({ + type: 'RECORDING_CONTROL', + action: 'start', + }) + } catch (err) { + setError('Failed to start recording') + setRecordingState('idle') + console.error('[useRecording] Failed to start recording:', err) + throw err + } + }, []) + + const stopRecording = useCallback(async (): Promise => { + setRecordingState('idle') + + // Tell background to stop + try { + await chrome.runtime.sendMessage({ + type: 'RECORDING_CONTROL', + action: 'stop', + }) + } catch (err) { + console.error('[useRecording] Failed to stop recording:', err) + } + + // Use ref to get the latest steps (avoids stale closure) + const currentSteps = stepsRef.current + if (currentSteps.length === 0) return null + + // Create and save recording (with empty name/desc initially) + try { + const recording = await saveRecording({ + v: 1, + name: '', + desc: '', + ts: startTimeRef.current, + startUrl: startUrlRef.current, + steps: currentSteps, + }) + + // Auto-name in background — don't block the UI + autoNameInBackground(recording) + + return recording + } catch (err) { + setError('Failed to save recording') + console.error('[useRecording] Failed to save recording:', err) + throw err + } + }, []) + + const discardRecording = useCallback(() => { + setRecordingState('idle') + setSteps([]) + setError(null) + rawEventsRef.current = [] + + chrome.runtime + .sendMessage({ + type: 'RECORDING_CONTROL', + action: 'stop', + }) + .catch(() => {}) + }, []) + + return { + recordingState, + steps, + startRecording, + stopRecording, + discardRecording, + eventCount: steps.length, + error, + } +} diff --git a/packages/extension/src/agent/useRecordingMention.ts b/packages/extension/src/agent/useRecordingMention.ts new file mode 100644 index 000000000..2ad4b20b7 --- /dev/null +++ b/packages/extension/src/agent/useRecordingMention.ts @@ -0,0 +1,230 @@ +/** + * useRecordingMention — Hook for @ mention recording in input + * + * Provides: + * - Parse @name from input text + * - Suggest matching recordings + * - Build replay task from mentioned recording + NL context + */ + +import { useCallback, useEffect, useRef, useState } from 'react' + +import { buildReplayTask } from '@/agent/RecordingReplayAgent' +import { listRecordings } from '@/lib/db' +import type { Recording } from '@/lib/recording-types' + +export interface MentionSuggestion { + recording: Recording + /** Display label: name or startUrl */ + label: string +} + +export interface RecordingMentionResult { + /** Current suggestions to show in dropdown */ + suggestions: MentionSuggestion[] + /** Whether the suggestion dropdown is visible */ + showSuggestions: boolean + /** The partial text after @ being typed */ + mentionQuery: string + /** Call when user selects a suggestion */ + selectSuggestion: (recording: Recording) => void + /** Call when input value changes */ + onInputChange: (value: string) => void + /** The resolved recording (after selection or from input) */ + resolvedRecording: Recording | null + /** Process input: if it contains a valid @mention, return replay task + system instruction */ + buildTaskFromInput: (inputValue: string) => Promise<{ + task: string + systemInstruction?: string + } | null> + /** Reset mention state */ + reset: () => void +} + +/** + * Extract @mention query from input text. + * Returns the text after the last @ symbol, or null if no active mention. + * + * Examples: + * "使用@搜索视频" → "搜索视频" + * "用@搜索 来搜AI" → null (space after @xxx = completed mention) + * "@B站搜索" → "B站搜索" + */ +function extractMentionQuery(text: string, cursorPos?: number): string | null { + const pos = cursorPos ?? text.length + const textBeforeCursor = text.slice(0, pos) + + // Find the last @ symbol + const atIndex = textBeforeCursor.lastIndexOf('@') + if (atIndex === -1) return null + + // The text after @ until cursor + const afterAt = textBeforeCursor.slice(atIndex + 1) + + // If there's a space after the mention text, it's completed + // But we should still track it for resolution + // Only return null if there's NO text after @ + if (afterAt.length === 0) return '' + + return afterAt +} + +/** + * Extract the mentioned recording name from input. + * This handles the completed mention case. + * + * "用@搜索视频 搜索AI" → "搜索视频" + * "@B站搜索 打开首页" → "B站搜索" + */ +function extractMentionedName(text: string): string | null { + const match = text.match(/@([^\s]+)/) + return match ? match[1] : null +} + +/** + * Extract NL instruction from input (everything except @mention). + * + * "用@搜索视频 搜索AI内容" → "用 搜索AI内容" + * "@登录B站 用账号xxx" → " 用账号xxx" + */ +function extractNLInstruction(text: string): string { + return text.replace(/@[^\s]+/, '').trim() +} + +export function useRecordingMention(): RecordingMentionResult { + const [suggestions, setSuggestions] = useState([]) + const [showSuggestions, setShowSuggestions] = useState(false) + const [mentionQuery, setMentionQuery] = useState('') + const [resolvedRecording, setResolvedRecording] = useState(null) + const allRecordingsRef = useRef([]) + const loadedRef = useRef(false) + + // Preload recordings list + useEffect(() => { + if (!loadedRef.current) { + loadedRef.current = true + listRecordings() + .then((recordings) => { + allRecordingsRef.current = recordings + }) + .catch(() => {}) + } + }, []) + + const onInputChange = useCallback((value: string) => { + const query = extractMentionQuery(value) + + if (query === null) { + setShowSuggestions(false) + setSuggestions([]) + setMentionQuery('') + return + } + + setMentionQuery(query) + + // Refresh recordings list every time @ is triggered (in case new recordings were added) + listRecordings() + .then((recordings) => { + allRecordingsRef.current = recordings + updateSuggestions(query, recordings) + }) + .catch(() => { + // Use cached list as fallback + updateSuggestions(query, allRecordingsRef.current) + }) + }, []) + + /** Filter and display suggestions based on query */ + function updateSuggestions(query: string, recordings: Recording[]) { + const filtered = recordings + .filter((r) => { + if (query === '') return true // Show all when just "@" + const label = r.name || r.startUrl || '' + return label.toLowerCase().includes(query.toLowerCase()) + }) + .slice(0, 6) + .map((r) => ({ + recording: r, + label: r.name || r.startUrl || 'Unnamed', + })) + + setSuggestions(filtered) + setShowSuggestions(filtered.length > 0) + } + + const selectSuggestion = useCallback((recording: Recording) => { + setResolvedRecording(recording) + setShowSuggestions(false) + setSuggestions([]) + setMentionQuery('') + }, []) + + const buildTaskFromInput = useCallback( + async (inputValue: string): Promise<{ task: string; systemInstruction?: string } | null> => { + // Try to find a recording reference + const mentionedName = extractMentionedName(inputValue) + if (!mentionedName) return null + + // Use resolved recording or try to find by name + let recording = resolvedRecording + if (!recording) { + // Reload recordings if needed + if (allRecordingsRef.current.length === 0) { + try { + allRecordingsRef.current = await listRecordings() + } catch { + return null + } + } + + recording = + allRecordingsRef.current.find( + (r) => + r.name === mentionedName || + r.name?.includes(mentionedName) || + r.startUrl?.includes(mentionedName) + ) ?? null + } + + if (!recording) return null + + // Extract the NL instruction (everything except @mention) + const nlInstruction = extractNLInstruction(inputValue) + + // Build replay task with NL modification + const { task, systemInstruction } = buildReplayTask( + recording, + undefined, // no param overrides from chat + nlInstruction || undefined + ) + + return { task, systemInstruction } + }, + [resolvedRecording] + ) + + const reset = useCallback(() => { + setResolvedRecording(null) + setShowSuggestions(false) + setSuggestions([]) + setMentionQuery('') + // Refresh recordings for next time + listRecordings() + .then((recordings) => { + allRecordingsRef.current = recordings + }) + .catch(() => {}) + }, []) + + return { + suggestions, + showSuggestions, + mentionQuery, + selectSuggestion, + onInputChange, + resolvedRecording, + buildTaskFromInput, + reset, + } +} diff --git a/packages/extension/src/components/MentionSuggestions.tsx b/packages/extension/src/components/MentionSuggestions.tsx new file mode 100644 index 000000000..a954ff589 --- /dev/null +++ b/packages/extension/src/components/MentionSuggestions.tsx @@ -0,0 +1,45 @@ +/** + * MentionSuggestions — Dropdown for @recording mention autocomplete + */ + +import type { MentionSuggestion } from '@/agent/useRecordingMention' +import type { Recording } from '@/lib/recording-types' + +export function MentionSuggestions({ + suggestions, + onSelect, +}: { + suggestions: MentionSuggestion[] + onSelect: (recording: Recording) => void +}) { + if (suggestions.length === 0) return null + + return ( +
+ {suggestions.map((s) => ( + + ))} +
+ ) +} diff --git a/packages/extension/src/components/ParamEditor.tsx b/packages/extension/src/components/ParamEditor.tsx new file mode 100644 index 000000000..39bbd6824 --- /dev/null +++ b/packages/extension/src/components/ParamEditor.tsx @@ -0,0 +1,69 @@ +import { useEffect, useState } from 'react' + +import { t } from '@/lib/i18n' + +/** + * ParamEditor — Renders editable input fields for all param-marked values + * + * Phase 6: i18n, accessibility (labels, fieldset) + */ +export function ParamEditor({ + params, + onChange, +}: { + params: Map + onChange: (overrides: Record) => void +}) { + const [values, setValues] = useState>(() => { + const initial: Record = {} + params.forEach((value, key) => { + initial[key] = value + }) + return initial + }) + + // Reset values when params change (e.g. navigating to a different recording) + useEffect(() => { + const updated: Record = {} + params.forEach((value, key) => { + updated[key] = value + }) + setValues(updated) + }, [params]) + + if (params.size === 0) return null + + const handleChange = (key: string, newValue: string) => { + const updated = { ...values, [key]: newValue } + setValues(updated) + onChange(updated) + } + + return ( +
+ + {t('params.title')} + + {Array.from(params.entries()).map(([key, defaultValue]) => ( +
+ + handleChange(key, e.target.value)} + placeholder={defaultValue} + className="flex-1 text-xs px-2 py-1 border rounded bg-background focus:outline-none focus:ring-1 focus:ring-ring" + aria-label={`${t('params.title')}: ${key}`} + /> +
+ ))} +
+ ) +} diff --git a/packages/extension/src/components/RecordingDetail.tsx b/packages/extension/src/components/RecordingDetail.tsx new file mode 100644 index 000000000..655c09eaf --- /dev/null +++ b/packages/extension/src/components/RecordingDetail.tsx @@ -0,0 +1,287 @@ +import { ArrowLeft, Copy, Pencil, Play, Save } from 'lucide-react' +import { useCallback, useEffect, useState } from 'react' + +import { buildReplayTask, extractParams } from '@/agent/RecordingReplayAgent' +import { Button } from '@/components/ui/button' +import { getRecording, updateRecording } from '@/lib/db' +import { t } from '@/lib/i18n' +import type { Recording } from '@/lib/recording-types' + +import { ParamEditor } from './ParamEditor' +import { RecordingStepCard } from './RecordingStepCard' + +/** + * RecordingDetail — Detail view for a single recording + * + * Features: + * - Edit name & description + * - View step list + * - Edit parameters + * - Natural language modification input + * - Export JSON + * - Replay button + * + * Phase 6: i18n, accessibility, error handling + */ +export function RecordingDetail({ + recordingId, + onBack, + onReplay, +}: { + recordingId: string + onBack: () => void + onReplay: (task: string, systemInstruction?: string) => void +}) { + const [recording, setRecording] = useState(null) + const [editing, setEditing] = useState(false) + const [name, setName] = useState('') + const [desc, setDesc] = useState('') + const [paramOverrides, setParamOverrides] = useState>({}) + const [nlMod, setNlMod] = useState('') + const [copied, setCopied] = useState(false) + const [error, setError] = useState(null) + + useEffect(() => { + // Reset all state when navigating to a different recording + setRecording(null) + setEditing(false) + setName('') + setDesc('') + setParamOverrides({}) + setNlMod('') + setCopied(false) + setError(null) + + getRecording(recordingId) + .then((r) => { + if (r) { + setRecording(r) + setName(r.name) + setDesc(r.desc) + } else { + setError(t('error.loadFailed')) + } + }) + .catch((err) => { + console.error('[RecordingDetail] Failed to load recording:', err) + setError(t('error.loadFailed')) + }) + }, [recordingId]) + + // Poll for auto-naming updates (LLM names in background after save) + useEffect(() => { + if (!recording || recording.name) return // already has a name, no need to poll + + const interval = setInterval(() => { + getRecording(recordingId) + .then((r) => { + if (r && r.name && r.name !== recording.name) { + setRecording(r) + setName(r.name) + setDesc(r.desc) + clearInterval(interval) + } + }) + .catch(() => {}) + }, 2000) + + // Stop polling after 30s max + const timeout = setTimeout(() => clearInterval(interval), 30000) + + return () => { + clearInterval(interval) + clearTimeout(timeout) + } + }, [recordingId, recording?.name]) + + const handleSave = useCallback(async () => { + if (!recording) return + try { + const updated = { ...recording, name, desc } + await updateRecording(updated) + setRecording(updated) + setEditing(false) + } catch (err) { + console.error('[RecordingDetail] Failed to save:', err) + } + }, [recording, name, desc]) + + const handleExportJSON = useCallback(async () => { + if (!recording) return + try { + const json = JSON.stringify(recording, null, 2) + await navigator.clipboard.writeText(json) + setCopied(true) + setTimeout(() => setCopied(false), 1500) + } catch (err) { + console.error('[RecordingDetail] Failed to copy JSON:', err) + } + }, [recording]) + + const handleReplay = useCallback(() => { + if (!recording) return + try { + const { task, systemInstruction } = buildReplayTask(recording, paramOverrides, nlMod) + onReplay(task, systemInstruction) + } catch (err) { + console.error('[RecordingDetail] Failed to build replay task:', err) + } + }, [recording, paramOverrides, nlMod, onReplay]) + + if (error) { + return ( +
+
+ + {t('detail.title')} +
+
+ {error} +
+
+ ) + } + + if (!recording) { + return ( +
+ {t('recordings.loading')} +
+ ) + } + + const params = extractParams(recording) + + return ( +
+ {/* Header */} +
+ + {t('detail.title')} + +
+ + {/* Content */} +
+ {/* Name & Description */} +
+ {editing ? ( + <> + setName(e.target.value)} + placeholder={t('detail.namePlaceholder')} + className="w-full text-xs font-medium px-2 py-1 border rounded bg-background focus:outline-none focus:ring-1 focus:ring-ring" + aria-label={t('detail.namePlaceholder')} + /> +