From b96a88e7487f97ef9c68c4d6cbbe771a5af35fa0 Mon Sep 17 00:00:00 2001 From: Arbaaz Ahmed Date: Wed, 22 Jul 2026 15:31:25 +0530 Subject: [PATCH 1/2] Draft PR for HTTP based copy and paste functionality --- package-lock.json | 44 ---------- src/hooks/useWebRtcStream.ts | 1 + src/routes/trackpad.tsx | 117 ++++++++++++++++++++++++-- src/server/api/InputPeerConnection.ts | 4 + src/server/api/apiHandlers.ts | 115 +++++++++++++++++++++++++ src/server/server.ts | 12 +++ 6 files changed, 243 insertions(+), 50 deletions(-) diff --git a/package-lock.json b/package-lock.json index 3708f2c..f526507 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9240,16 +9240,6 @@ } } }, - "node_modules/nitro-nightly/node_modules/lru-cache": { - "version": "11.5.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", - "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", - "extraneous": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/nitro-nightly/node_modules/srvx": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/srvx/-/srvx-0.10.1.tgz", @@ -9364,16 +9354,6 @@ } } }, - "node_modules/nitro/node_modules/lru-cache": { - "version": "11.5.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.1.tgz", - "integrity": "sha512-RPimw/7aMdv2oqRrxKwvZXcPfwBrn/JZ2xYcY9Hus/6LaS3VOAKVWKWgNLCFSiOm1ESXinjsDlidVU7JlnCN2A==", - "extraneous": true, - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, "node_modules/nitro/node_modules/unstorage": { "version": "2.0.0-alpha.7", "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-2.0.0-alpha.7.tgz", @@ -12818,30 +12798,6 @@ "node": ">=18" } }, - "node_modules/xml2js": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", - "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", - "extraneous": true, - "license": "MIT", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xml2js/node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "extraneous": true, - "license": "MIT", - "engines": { - "node": ">=4.0" - } - }, "node_modules/xmlbuilder": { "version": "15.1.1", "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", diff --git a/src/hooks/useWebRtcStream.ts b/src/hooks/useWebRtcStream.ts index bef9158..b826270 100644 --- a/src/hooks/useWebRtcStream.ts +++ b/src/hooks/useWebRtcStream.ts @@ -467,5 +467,6 @@ export function useWebRtcStream({ token }: UseWebRtcStreamOptions) { errorHandle, reconnect, sendInputEvent, + activeSessionId, } } diff --git a/src/routes/trackpad.tsx b/src/routes/trackpad.tsx index 9d06217..99b332c 100644 --- a/src/routes/trackpad.tsx +++ b/src/routes/trackpad.tsx @@ -11,6 +11,40 @@ import { ScreenMirror } from "../components/Trackpad/ScreenMirror" import { ErrorComponent } from "../components/Trackpad/ErrorComponent" import { useWebRtcStream } from "../hooks/useWebRtcStream" +const copyWithFallback = (text: string) => { + const textArea = document.createElement("textarea") + textArea.value = text + textArea.setAttribute("readonly", "") + textArea.style.position = "absolute" + textArea.style.left = "-9999px" + document.body.appendChild(textArea) + textArea.select() + textArea.setSelectionRange(0, text.length) + try { + return document.execCommand("copy") + } finally { + document.body.removeChild(textArea) + } +} +const writeClientClipboard = async (text: string) => { + if ( + typeof navigator !== "undefined" && + navigator.clipboard && + typeof navigator.clipboard.writeText === "function" + ) { + try { + await navigator.clipboard.writeText(text) + return + } catch (err) { + console.warn("navigator.clipboard.writeText failed, using fallback:", err) + } + } + const success = copyWithFallback(text) + if (!success) { + throw new Error("Fallback copy failed") + } +} + export const Route = createFileRoute("/trackpad")({ component: TrackpadPage, }) @@ -43,10 +77,16 @@ function TrackpadPage() { const [keyboardOpen, setKeyboardOpen] = useState(false) const [extraKeysVisible, setExtraKeysVisible] = useState(true) const { status, send, sendCombo } = useRemoteConnection() - const { trackActive, videoStream, error, errorHandle, reconnect } = - useWebRtcStream({ - token, - }) + const { + trackActive, + videoStream, + error, + errorHandle, + reconnect, + activeSessionId, + } = useWebRtcStream({ + token, + }) // Send input actions safely over WebRTC DataChannels const broadcastMessage = (payload: unknown) => { @@ -75,8 +115,73 @@ function TrackpadPage() { ) } - const handleCopy = () => broadcastMessage({ type: "copy" }) - const handlePaste = async () => broadcastMessage({ type: "paste" }) + const handleCopy = async () => { + try { + const headers: Record = {} + if (token) { + headers.Authorization = `Bearer ${token}` + } + const response = await fetch("/api/clipboard/copy", { + method: "POST", + headers: { + ...headers, + "Content-Type": "application/json", + }, + body: JSON.stringify({ sessionId: activeSessionId }), + }) + if (response.ok) { + const data = await response.json() + if (data && typeof data.text === "string") { + await writeClientClipboard(data.text) + } else { + throw new Error("Invalid copy response data") + } + } else { + throw new Error(`Clipboard copy failed: ${response.statusText}`) + } + } catch (err) { + console.warn( + "Client clipboard copy failed, falling back to server copy:", + err, + ) + broadcastMessage({ type: "copy" }) + } + } + const handlePaste = async () => { + try { + if ( + typeof navigator !== "undefined" && + navigator.clipboard && + typeof navigator.clipboard.readText === "function" + ) { + const text = await navigator.clipboard.readText() + if (text) { + const headers: Record = {} + if (token) { + headers.Authorization = `Bearer ${token}` + } + const response = await fetch("/api/clipboard/paste", { + method: "POST", + headers: { + ...headers, + "Content-Type": "application/json", + }, + body: JSON.stringify({ sessionId: activeSessionId, text }), + }) + if (response.ok) { + return + } + } + } + throw new Error("Client clipboard read returned empty or is unavailable") + } catch (err) { + console.warn( + "Client clipboard paste failed, falling back to server clipboard:", + err, + ) + broadcastMessage({ type: "paste" }) + } + } const handleInput = (e: React.ChangeEvent) => { const nativeEvent = e.nativeEvent as InputEvent diff --git a/src/server/api/InputPeerConnection.ts b/src/server/api/InputPeerConnection.ts index 0d50823..0fb2a33 100644 --- a/src/server/api/InputPeerConnection.ts +++ b/src/server/api/InputPeerConnection.ts @@ -150,6 +150,10 @@ export class InputPeerConnection { this.inputHandler.updateConfig(config) } + async handleMessage(msg: InputMessage): Promise { + await this.inputHandler.handleMessage(msg) + } + close(): void { try { this.pc.close() diff --git a/src/server/api/apiHandlers.ts b/src/server/api/apiHandlers.ts index f71015f..944160a 100644 --- a/src/server/api/apiHandlers.ts +++ b/src/server/api/apiHandlers.ts @@ -3,6 +3,8 @@ */ import type { IncomingMessage, ServerResponse } from "node:http" +import { spawnSync } from "node:child_process" +import os from "node:os" import fs from "node:fs" import crypto from "node:crypto" import logger from "../../utils/logger" @@ -752,3 +754,116 @@ export async function handleWhipSignalingExchange( }, 100) req.on("close", () => clearInterval(answerCheckInterval)) } + +function writeHostClipboard(text: string): void { + const platform = os.platform() + if (platform === "darwin") { + spawnSync("pbcopy", { input: text }) + } else if (platform === "win32") { + spawnSync("clip", { input: text }) + } else { + const procWl = spawnSync("wl-copy", { input: text }) + if (procWl.status !== 0) { + const procXclip = spawnSync("xclip", ["-selection", "clipboard"], { + input: text, + }) + if (procXclip.status !== 0) { + spawnSync("xsel", ["--clipboard", "--input"], { input: text }) + } + } + } +} +function readHostClipboard(): string { + const platform = os.platform() + if (platform === "darwin") { + const proc = spawnSync("pbpaste", { encoding: "utf-8" }) + return proc.stdout || "" + } else if (platform === "win32") { + const proc = spawnSync( + "powershell", + ["-NoProfile", "-Command", "Get-Clipboard"], + { encoding: "utf-8" }, + ) + return (proc.stdout || "").replace(/\r\n$/, "").replace(/\n$/, "") + } else { + const procWl = spawnSync("wl-paste", ["--no-newline"], { + encoding: "utf-8", + }) + if (procWl.status === 0) { + return procWl.stdout || "" + } + const procXclip = spawnSync("xclip", ["-selection", "clipboard", "-o"], { + encoding: "utf-8", + }) + if (procXclip.status === 0) { + return procXclip.stdout || "" + } + const procXsel = spawnSync("xsel", ["--clipboard", "--output"], { + encoding: "utf-8", + }) + if (procXsel.status === 0) { + return procXsel.stdout || "" + } + } + return "" +} +export async function handleClipboardCopy( + req: IncomingMessage, + res: ServerResponse, +): Promise { + if (!requireAuth(req, res)) return + const bodyText = await readBody(req) + const { sessionId } = JSON.parse(bodyText || "{}") as { + sessionId?: string + } + if (!sessionId) { + json(res, 400, { error: "sessionId is required" }) + return + } + const inputPc = inputConnections.get(sessionId) + if (!inputPc) { + json(res, 404, { error: "Input connection not active" }) + return + } + await inputPc.handleMessage({ type: "copy" }) + await new Promise((resolve) => setTimeout(resolve, 100)) + try { + const text = readHostClipboard() + json(res, 200, { text }) + } catch (err) { + logger.error(`Failed to read host clipboard: ${String(err)}`) + json(res, 500, { error: "Failed to read host clipboard" }) + } +} +export async function handleClipboardPaste( + req: IncomingMessage, + res: ServerResponse, +): Promise { + if (!requireAuth(req, res)) return + const bodyText = await readBody(req) + const { sessionId, text } = JSON.parse(bodyText || "{}") as { + sessionId?: string + text?: string + } + if (!sessionId || typeof text !== "string") { + json(res, 400, { error: "sessionId and text are required" }) + return + } + const inputPc = inputConnections.get(sessionId) + if (!inputPc) { + json(res, 404, { error: "Input connection not active" }) + return + } + try { + writeHostClipboard(text) + } catch (err) { + logger.error(`Failed to write host clipboard (non-fatal): ${String(err)}`) + } + try { + await inputPc.handleMessage({ type: "text", text }) + json(res, 200, { ok: true }) + } catch (err) { + logger.error(`Failed to inject text: ${String(err)}`) + json(res, 500, { error: "Failed to inject pasted text" }) + } +} diff --git a/src/server/server.ts b/src/server/server.ts index 637aeaf..34cd0a6 100644 --- a/src/server/server.ts +++ b/src/server/server.ts @@ -23,6 +23,8 @@ import { handleWhipSignalingExchange, handleGetIp, handleUpdateConfig, + handleClipboardCopy, + handleClipboardPaste, json, } from "./api/apiHandlers" @@ -82,6 +84,16 @@ const routes: Route[] = [ pattern: /^\/api\/webrtc\/whip$/, handler: handleWhipSignalingExchange, }, + { + method: "POST", + pattern: /^\/api\/clipboard\/copy$/, + handler: handleClipboardCopy, + }, + { + method: "POST", + pattern: /^\/api\/clipboard\/paste$/, + handler: handleClipboardPaste, + }, ] export function attachSignalingRoutes( From 0c39a0cc95dfedd8edd62e0e4caa9d60fde4b289 Mon Sep 17 00:00:00 2001 From: Arbaaz Ahmed Date: Tue, 28 Jul 2026 10:23:42 +0530 Subject: [PATCH 2/2] fix: improve Unicode text injection for HTTP clipboard --- src/server/api/apiHandlers.ts | 21 ++++++++++++++++----- src/server/drivers/linux/keyboard.ts | 2 +- src/server/drivers/mac/keyboard.ts | 23 ++++------------------- src/server/drivers/windows/keyboard.ts | 2 +- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/server/api/apiHandlers.ts b/src/server/api/apiHandlers.ts index 944160a..850383a 100644 --- a/src/server/api/apiHandlers.ts +++ b/src/server/api/apiHandlers.ts @@ -773,35 +773,46 @@ function writeHostClipboard(text: string): void { } } } +// 100 MiB — far beyond any realistic clipboard content; prevents spawnSync +// from silently truncating stdout when the default 1 MiB maxBuffer is exceeded. +const CLIPBOARD_MAX_BUFFER = 100 * 1024 * 1024 function readHostClipboard(): string { const platform = os.platform() if (platform === "darwin") { - const proc = spawnSync("pbpaste", { encoding: "utf-8" }) + const proc = spawnSync("pbpaste", { + encoding: "utf-8", + maxBuffer: CLIPBOARD_MAX_BUFFER, + }) + if (proc.error) throw proc.error return proc.stdout || "" } else if (platform === "win32") { const proc = spawnSync( "powershell", ["-NoProfile", "-Command", "Get-Clipboard"], - { encoding: "utf-8" }, + { encoding: "utf-8", maxBuffer: CLIPBOARD_MAX_BUFFER }, ) + if (proc.error) throw proc.error return (proc.stdout || "").replace(/\r\n$/, "").replace(/\n$/, "") } else { const procWl = spawnSync("wl-paste", ["--no-newline"], { encoding: "utf-8", + maxBuffer: CLIPBOARD_MAX_BUFFER, }) - if (procWl.status === 0) { + if (procWl.status === 0 && !procWl.error) { return procWl.stdout || "" } const procXclip = spawnSync("xclip", ["-selection", "clipboard", "-o"], { encoding: "utf-8", + maxBuffer: CLIPBOARD_MAX_BUFFER, }) - if (procXclip.status === 0) { + if (procXclip.status === 0 && !procXclip.error) { return procXclip.stdout || "" } const procXsel = spawnSync("xsel", ["--clipboard", "--output"], { encoding: "utf-8", + maxBuffer: CLIPBOARD_MAX_BUFFER, }) - if (procXsel.status === 0) { + if (procXsel.status === 0 && !procXsel.error) { return procXsel.stdout || "" } } diff --git a/src/server/drivers/linux/keyboard.ts b/src/server/drivers/linux/keyboard.ts index 7f5aa2a..d1d2754 100644 --- a/src/server/drivers/linux/keyboard.ts +++ b/src/server/drivers/linux/keyboard.ts @@ -33,7 +33,7 @@ export class LinuxKeyboard { this.sendKeyEvent(code, KEY_RELEASE) } this.sync() - } else if (key.length === 1) { + } else if (key.length > 0) { this.injectText(key) } else { console.warn("[LinuxKeyboard] Unknown key:", key) diff --git a/src/server/drivers/mac/keyboard.ts b/src/server/drivers/mac/keyboard.ts index 3198b9d..b9e9cff 100644 --- a/src/server/drivers/mac/keyboard.ts +++ b/src/server/drivers/mac/keyboard.ts @@ -1,10 +1,4 @@ -/** - * macOS virtual keyboard implementation. - * - * Handles key, key-combination, and text injection through CoreGraphics - * keyboard events. Supports both key-code based input and Unicode - * character injection for characters not present in the standard key map. - */ +import koffi from "koffi" import { postKeyEvent, postMediaKeyEvent, @@ -39,7 +33,7 @@ export class MacKeyboard { if (code !== undefined) { if (pos !== "RELEASE") postKeyEvent(code, true) if (pos !== "HOLD") postKeyEvent(code, false) - } else if (key.length === 1) { + } else if (key.length > 0) { this.injectText(key) } else { console.warn("[MacKeyboard] Unknown key:", key) @@ -72,20 +66,13 @@ export class MacKeyboard { if (!text) return for (const ch of text) { const { code, shifted } = resolveChar(ch, MAC_KEY_MAP) - const shiftCode = MAC_KEY_MAP.shift - if (code === undefined) { - // Fall back to Unicode injection for unmapped characters. + if (code === undefined || shifted) { + // Fall back to Unicode injection for unmapped or shifted characters. this.injectUnicodeChar(ch) continue } - if (shiftCode === undefined) { - console.warn("[MacKeyboard] Shift key code not defined in key map") - continue - } - if (shifted) postKeyEvent(shiftCode, true) postKeyEvent(code, true) postKeyEvent(code, false) - if (shifted) postKeyEvent(shiftCode, false) } } private injectUnicodeChar(ch: string): void { @@ -108,7 +95,6 @@ function ensureUnicode() { if (_unicodeInjectorLoaded) return _unicodeInjectorLoaded = true try { - const koffi = require("koffi") const lib = koffi.load( "/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics", ) @@ -148,7 +134,6 @@ function injectUnicode(ch: string): void { const upRef = _CGEventCreateKeyboardEvent(null, 0, 0) if (!upRef) return - _CGEventKeyboardSetUnicodeString(upRef, charCount, buf) _CGEventPost(0, upRef) _CFRelease(upRef) } diff --git a/src/server/drivers/windows/keyboard.ts b/src/server/drivers/windows/keyboard.ts index 19cdf6e..a5e8120 100644 --- a/src/server/drivers/windows/keyboard.ts +++ b/src/server/drivers/windows/keyboard.ts @@ -40,7 +40,7 @@ export class WindowsKeyboard { }) } this.sendInput(events.length, events) - } else if (key.length === 1) { + } else if (key.length > 0) { this.injectText(key) } else { console.warn("[Keyboard] Unknown key and not a single character:", key)