From 867359c0c269731eff786793fc26be0afc763fa0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Mar 2026 14:41:31 +0000 Subject: [PATCH 1/2] Initial plan From d009f2f8bf049e66af6cef55379bed14a554e594 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 23 Mar 2026 14:52:51 +0000 Subject: [PATCH 2/2] feat: streaming animation player, FACS slider panel, streaming demo Co-authored-by: hmthanh <8927701+hmthanh@users.noreply.github.com> Agent-Logs-Url: https://github.com/openhuman-ai/realistic-render-engine/sessions/d07721e8-4ffb-46f0-8203-539f7408286e --- demo/index.html | 222 +++++++ demo/streaming-server.js | 92 +++ demo/streaming.html | 747 ++++++++++++++++++++++ src/animation/StreamingAnimationPlayer.js | 643 +++++++++++++++++++ src/sdk/OpenHuman.js | 127 +++- 5 files changed, 1826 insertions(+), 5 deletions(-) create mode 100644 demo/streaming-server.js create mode 100644 demo/streaming.html create mode 100644 src/animation/StreamingAnimationPlayer.js diff --git a/demo/index.html b/demo/index.html index 9d9635f..50c7bea 100644 --- a/demo/index.html +++ b/demo/index.html @@ -184,6 +184,130 @@ padding: 24px; text-align: center; } + + /* ── FACS Slider Panel (left side, PR #5) */ + #facs-panel { + position: absolute; + top: 12px; + left: 14px; + width: 230px; + max-height: calc(100% - 100px); + background: rgba(10,12,16,0.88); + border: 1px solid #2a2e38; + border-radius: 8px; + display: flex; + flex-direction: column; + font-size: 0.72rem; + color: #c8ccd8; + overflow: hidden; + z-index: 20; + /* Hide by default β€” toggled by #btn-facs */ + transform: translateX(-110%); + transition: transform 0.22s ease; + } + #facs-panel.open { transform: translateX(0); } + + #facs-panel-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 8px 12px; + background: rgba(255,255,255,0.04); + border-bottom: 1px solid #2a2e38; + flex-shrink: 0; + } + #facs-panel-header span { + font-size: 0.68rem; + text-transform: uppercase; + letter-spacing: 0.08em; + color: #8a9; + font-weight: 600; + } + #btn-facs-reset { + font-size: 0.65rem; + padding: 2px 8px; + background: rgba(60,30,30,0.8); + border: 1px solid #7a3a3a; + border-radius: 4px; + color: #ffaaaa; + cursor: pointer; + } + #btn-facs-reset:hover { background: rgba(90,40,40,0.9); } + + #facs-scroll { + overflow-y: auto; + flex: 1; + padding: 6px 0; + scrollbar-width: thin; + scrollbar-color: #333 transparent; + } + #facs-scroll::-webkit-scrollbar { width: 4px; } + #facs-scroll::-webkit-scrollbar-thumb { background: #333; border-radius: 2px; } + + .facs-group-label { + font-size: 0.6rem; + text-transform: uppercase; + letter-spacing: 0.1em; + color: #556; + padding: 6px 12px 2px; + border-top: 1px solid #1e2228; + margin-top: 4px; + } + .facs-group-label:first-child { border-top: none; margin-top: 0; } + + .facs-row { + display: flex; + align-items: center; + gap: 6px; + padding: 2px 10px 2px 12px; + } + .facs-row label { + flex: 1; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + color: #aab; + font-size: 0.68rem; + cursor: default; + } + .facs-row input[type="range"] { + width: 70px; + accent-color: #7ec8f8; + cursor: pointer; + height: 3px; + } + .facs-val { + width: 26px; + text-align: right; + color: #7ec8f8; + font-size: 0.66rem; + font-variant-numeric: tabular-nums; + flex-shrink: 0; + } + + /* Toggle button for FACS panel β€” bottom-left */ + #btn-facs-toggle { + position: absolute; + bottom: 60px; + left: 14px; + font-size: 0.72rem; + padding: 5px 12px; + background: rgba(20,50,80,0.9); + border: 1px solid #2a6aaf; + border-radius: 4px; + color: #7ec8f8; + cursor: pointer; + z-index: 21; + } + #btn-facs-toggle:hover { background: rgba(30,70,110,0.95); } + + /* Shift HUD down when FACS panel is open so they don't overlap */ + #facs-panel.open ~ #hud { top: auto; bottom: 90px; } + + @media (max-width: 600px) { + #facs-panel { width: 190px; } + .facs-row input[type="range"] { width: 52px; } + } @@ -302,6 +426,20 @@

OpenHuman Engine

+ +
+
+ FACS Blendshapes + +
+
+ +
+
+ + + +
πŸ–± Drag to orbit  Β·  Scroll to zoom  Β·  Touch supported
@@ -844,6 +982,90 @@

OpenHuman Engine

setTimeout(() => { if (stateEl.textContent.startsWith('talk')) stateEl.textContent = 'talk'; }, 380); }); + // ── FACS Slider Panel (PR #5) + // Groups matching the canonical FACS_NAMES order + const FACS_GROUPS = [ + { label: 'Brow', names: ['browDownLeft','browDownRight','browInnerUp','browOuterUpLeft','browOuterUpRight'] }, + { label: 'Cheek', names: ['cheekPuff','cheekSquintLeft','cheekSquintRight'] }, + { label: 'Eye β€” Blink & Squint', names: ['eyeBlinkLeft','eyeBlinkRight','eyeSquintLeft','eyeSquintRight','eyeWideLeft','eyeWideRight'] }, + { label: 'Eye β€” Look', names: ['eyeLookDownLeft','eyeLookDownRight','eyeLookInLeft','eyeLookInRight','eyeLookOutLeft','eyeLookOutRight','eyeLookUpLeft','eyeLookUpRight'] }, + { label: 'Jaw', names: ['jawForward','jawLeft','jawRight','jawOpen','mouthClose'] }, + { label: 'Mouth β€” Shape', names: ['mouthFunnel','mouthPucker','mouthLeft','mouthRight','mouthRollLower','mouthRollUpper','mouthShrugLower','mouthShrugUpper'] }, + { label: 'Mouth β€” Smile / Frown', names: ['mouthSmileLeft','mouthSmileRight','mouthFrownLeft','mouthFrownRight','mouthDimpleLeft','mouthDimpleRight'] }, + { label: 'Mouth β€” Stretch / Press', names: ['mouthStretchLeft','mouthStretchRight','mouthPressLeft','mouthPressRight'] }, + { label: 'Mouth β€” Upper / Lower', names: ['mouthUpperUpLeft','mouthUpperUpRight','mouthLowerDownLeft','mouthLowerDownRight'] }, + { label: 'Nose', names: ['noseSneerLeft','noseSneerRight'] }, + { label: 'Tongue', names: ['tongueOut'] }, + ]; + + /** Build the FACS panel rows and wire slider events. */ + function buildFACSPanel() { + const scroll = document.getElementById('facs-scroll'); + scroll.innerHTML = ''; + const sliderMap = new Map(); // name β†’ { slider, valEl } + + for (const group of FACS_GROUPS) { + const groupEl = document.createElement('div'); + groupEl.className = 'facs-group-label'; + groupEl.textContent = group.label; + scroll.appendChild(groupEl); + + for (const name of group.names) { + const row = document.createElement('div'); + row.className = 'facs-row'; + + const label = document.createElement('label'); + label.title = name; + // Shorten camelCase for display: insert spaces before uppercase + label.textContent = name.replace(/([A-Z])/g, ' $1').trim(); + + const slider = document.createElement('input'); + slider.type = 'range'; + slider.min = '0'; + slider.max = '1'; + slider.step = '0.01'; + slider.value = '0'; + + const valEl = document.createElement('span'); + valEl.className = 'facs-val'; + valEl.textContent = '0.00'; + + slider.addEventListener('input', () => { + const w = parseFloat(slider.value); + valEl.textContent = w.toFixed(2); + morphCtrl.set(name, w); + }); + + row.appendChild(label); + row.appendChild(slider); + row.appendChild(valEl); + scroll.appendChild(row); + sliderMap.set(name, { slider, valEl }); + } + } + + // Reset all + document.getElementById('btn-facs-reset').addEventListener('click', () => { + for (const [name, { slider, valEl }] of sliderMap) { + slider.value = '0'; + valEl.textContent = '0.00'; + morphCtrl.set(name, 0); + } + }); + + return sliderMap; + } + + buildFACSPanel(); + + // FACS panel open/close toggle + const facsPanel = document.getElementById('facs-panel'); + const facsBtnTgl = document.getElementById('btn-facs-toggle'); + facsBtnTgl.addEventListener('click', () => { + facsPanel.classList.toggle('open'); + facsBtnTgl.textContent = facsPanel.classList.contains('open') ? 'βœ• Close FACS' : '🎭 FACS'; + }); + // ── Render loop let lastTs = performance.now(); let frameCount = 0; diff --git a/demo/streaming-server.js b/demo/streaming-server.js new file mode 100644 index 0000000..b9c49d7 --- /dev/null +++ b/demo/streaming-server.js @@ -0,0 +1,92 @@ +#!/usr/bin/env node +/** + * streaming-server.js β€” Minimal WebSocket test server for the OpenHuman streaming demo. + * + * Sends synthetic animation frames at ~30 FPS over WebSocket using the binary protocol + * documented in src/animation/StreamingAnimationPlayer.js. + * + * Usage: + * npm install ws # one-time dev dependency + * node demo/streaming-server.js + * # Then open demo/streaming.html and connect to ws://localhost:8765 + * + * Binary frame layout (little-endian): + * [0..3] uint32 serverTimestampMs + * [4..7] uint32 frameId + * [8] uint8 jointCount + * [9] uint8 facsCount + * [10..11] uint16 flags (0) + * then jointCount Γ— 14 bytes (3Γ—int16 pos + 4Γ—int16 quat) + * then facsCount Γ— 2 bytes (int16 weight Γ— 32767) + */ + +'use strict'; + +const { WebSocketServer } = require('ws'); +const PORT = 8765; +const JOINT_COUNT = 3; +const FACS_COUNT = 52; +const FPS = 30; +const FRAME_MS = 1000 / FPS; + +const FRAME_SIZE = 12 + JOINT_COUNT * 14 + FACS_COUNT * 2; + +let frameId = 0; +let startMs = Date.now(); +const frameBuf = Buffer.allocUnsafe(FRAME_SIZE); +const view = new DataView(frameBuf.buffer); + +function buildFrame() { + const elapsed = (Date.now() - startMs) / 1000; + const serverTs = (Date.now()) >>> 0; + + view.setUint32(0, serverTs, true); + view.setUint32(4, frameId++ >>> 0, true); + view.setUint8(8, JOINT_COUNT); + view.setUint8(9, FACS_COUNT); + view.setUint16(10, 0, true); + + let off = 12; + for (let j = 0; j < JOINT_COUNT; j++) { + // Position (mm int16) + view.setInt16(off, Math.round(Math.sin(elapsed * 0.7 + j * 1.2) * 50), true); off += 2; + view.setInt16(off, Math.round(j * 700), true); off += 2; + view.setInt16(off, Math.round(Math.cos(elapsed * 0.5 + j * 0.9) * 40), true); off += 2; + // Rotation (normalized quaternion Γ— 32767) + const angle = elapsed * (0.8 + j * 0.3); + const qy = Math.sin(angle * 0.5), qw = Math.cos(angle * 0.5); + view.setInt16(off, 0, true); off += 2; + view.setInt16(off, Math.round(qy * 32767), true); off += 2; + view.setInt16(off, 0, true); off += 2; + view.setInt16(off, Math.round(qw * 32767), true); off += 2; + } + for (let f = 0; f < FACS_COUNT; f++) { + const w = Math.max(0, Math.sin(elapsed * (0.3 + f * 0.07) + f)) * 0.5; + view.setInt16(off, Math.round(Math.max(0, Math.min(1, w)) * 32767), true); off += 2; + } + return frameBuf; +} + +const wss = new WebSocketServer({ port: PORT }); +console.log(`[streaming-server] Listening on ws://localhost:${PORT}`); + +const clients = new Set(); +wss.on('connection', ws => { + clients.add(ws); + console.log(`[streaming-server] Client connected (total: ${clients.size})`); + ws.on('close', () => { + clients.delete(ws); + console.log(`[streaming-server] Client disconnected (total: ${clients.size})`); + }); + ws.on('error', err => console.error('[streaming-server] client error:', err.message)); +}); + +setInterval(() => { + if (clients.size === 0) return; + const buf = buildFrame(); + for (const ws of clients) { + if (ws.readyState === 1 /* OPEN */) { + ws.send(buf); + } + } +}, FRAME_MS); diff --git a/demo/streaming.html b/demo/streaming.html new file mode 100644 index 0000000..7106cf1 --- /dev/null +++ b/demo/streaming.html @@ -0,0 +1,747 @@ + + + + + + OpenHuman Engine β€” Streaming Demo + + + + +
+

OpenHuman Engine

+ WebGL 2.0 + Zero deps + Streaming + Disconnected +
+ +
+ + +
+ + + +
+
Render FPS: β€”
+
Stream FPS: β€”
+
Latency: β€” ms
+
Dropped: 0
+
Buffer: 0 frames
+
+ +
+
+
+
+ + + +
+ + + + diff --git a/src/animation/StreamingAnimationPlayer.js b/src/animation/StreamingAnimationPlayer.js new file mode 100644 index 0000000..9061c62 --- /dev/null +++ b/src/animation/StreamingAnimationPlayer.js @@ -0,0 +1,643 @@ +/** + * StreamingAnimationPlayer β€” real-time animation streaming via WebSocket or HTTP. + * + * Binary frame protocol (little-endian): + * + * Header β€” 12 bytes: + * [0..3] uint32 serverTimestampMs β€” server wall-clock time for this frame + * [4..7] uint32 frameId β€” monotonically increasing frame counter + * [8] uint8 jointCount β€” number of joints encoded + * [9] uint8 facsCount β€” number of FACS weights encoded (≀ 52) + * [10..11] uint16 flags β€” reserved, must be 0 + * + * Joint data β€” jointCount Γ— 14 bytes each: + * [0..5] 3 Γ— int16 position (metres Γ— 1 000 β€” i.e. millimetre precision) + * [6..13] 4 Γ— int16 quaternion (normalized; each component Γ— 32 767) + * + * FACS data β€” facsCount Γ— 2 bytes each: + * int16 morph weight (Γ— 32 767; clamped 0..1 = 0..32767) + * + * Jitter buffer: + * Incoming frames are timestamped with the *local* arrival time. + * A configurable _targetDelayMs (default 60 ms) is added to form a + * "playback clock" that runs slightly behind the live edge. + * getInterpolatedPose(nowMs) returns a pose blended between the two + * frames that bracket the playback position. When the buffer is + * exhausted (packet loss / stall) the last good pose is returned + * (hold/extrapolate) to avoid visual pops. + * + * Zero runtime dependencies. + */ + +export const STREAM_TYPE_WS = 'ws'; +export const STREAM_TYPE_HTTP = 'http'; + +/** Maximum frames held in the jitter buffer before oldest are evicted. */ +const BUFFER_MAX_FRAMES = 64; + +/** Quantisation scales matching the binary protocol. */ +const POS_SCALE = 1 / 1000; // int16 β†’ metres +const NORM_SCALE = 1 / 32767; // int16 β†’ normalized float (joints + FACS) + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +/** Decode a binary frame ArrayBuffer into a frame object. */ +function decodeFrame(buffer) { + if (buffer.byteLength < 12) { + throw new RangeError(`[StreamingAnimationPlayer] Frame too short: ${buffer.byteLength} bytes`); + } + const view = new DataView(buffer); + const serverTs = view.getUint32(0, true); + const frameId = view.getUint32(4, true); + const jointCount = view.getUint8(8); + const facsCount = view.getUint8(9); + // flags at [10..11] reserved + + const minLength = 12 + jointCount * 14 + facsCount * 2; + if (buffer.byteLength < minLength) { + throw new RangeError( + `[StreamingAnimationPlayer] Frame data too short: expected β‰₯${minLength}, got ${buffer.byteLength}` + ); + } + + // Decode joint translations & rotations + const joints = new Float32Array(jointCount * 7); // tx,ty,tz, qx,qy,qz,qw per joint + let offset = 12; + for (let j = 0; j < jointCount; j++) { + const base = j * 7; + joints[base] = view.getInt16(offset, true) * POS_SCALE; + joints[base + 1] = view.getInt16(offset + 2, true) * POS_SCALE; + joints[base + 2] = view.getInt16(offset + 4, true) * POS_SCALE; + joints[base + 3] = view.getInt16(offset + 6, true) * NORM_SCALE; + joints[base + 4] = view.getInt16(offset + 8, true) * NORM_SCALE; + joints[base + 5] = view.getInt16(offset + 10, true) * NORM_SCALE; + joints[base + 6] = view.getInt16(offset + 12, true) * NORM_SCALE; + offset += 14; + } + + // Decode FACS weights + const facs = new Float32Array(facsCount); + for (let f = 0; f < facsCount; f++) { + facs[f] = Math.max(0, view.getInt16(offset, true) * NORM_SCALE); + offset += 2; + } + + return { serverTs, frameId, joints, facs, arrivalTs: performance.now() }; +} + +/** Encode a frame to binary (useful for mock server / testing). */ +export function encodeFrame(frame) { + const { serverTs, frameId, joints, facs } = frame; + const jointCount = joints.length / 7 | 0; + const facsCount = facs.length; + const buf = new ArrayBuffer(12 + jointCount * 14 + facsCount * 2); + const view = new DataView(buf); + + view.setUint32(0, serverTs >>> 0, true); + view.setUint32(4, frameId >>> 0, true); + view.setUint8(8, jointCount); + view.setUint8(9, facsCount); + view.setUint16(10, 0, true); // flags + + let offset = 12; + for (let j = 0; j < jointCount; j++) { + const base = j * 7; + view.setInt16(offset, Math.round(joints[base] / POS_SCALE), true); + view.setInt16(offset + 2, Math.round(joints[base + 1] / POS_SCALE), true); + view.setInt16(offset + 4, Math.round(joints[base + 2] / POS_SCALE), true); + view.setInt16(offset + 6, Math.round(joints[base + 3] / NORM_SCALE), true); + view.setInt16(offset + 8, Math.round(joints[base + 4] / NORM_SCALE), true); + view.setInt16(offset + 10, Math.round(joints[base + 5] / NORM_SCALE), true); + view.setInt16(offset + 12, Math.round(joints[base + 6] / NORM_SCALE), true); + offset += 14; + } + for (let f = 0; f < facsCount; f++) { + view.setInt16(offset, Math.round(Math.max(0, Math.min(1, facs[f])) * 32767), true); + offset += 2; + } + return buf; +} + +/** Linear interpolation between two float arrays of equal length. */ +function lerpArray(a, b, t, out) { + for (let i = 0; i < a.length; i++) { + out[i] = a[i] + (b[i] - a[i]) * t; + } +} + +/** Spherical linear interpolation for a quaternion stored at offset in a Float32Array. */ +function slerpJointQuat(a, b, t, out, base) { + let ax = a[base + 3], ay = a[base + 4], az = a[base + 5], aw = a[base + 6]; + let bx = b[base + 3], by = b[base + 4], bz = b[base + 5], bw = b[base + 6]; + + let dot = ax * bx + ay * by + az * bz + aw * bw; + if (dot < 0) { bx = -bx; by = -by; bz = -bz; bw = -bw; dot = -dot; } + + let s0, s1; + if (dot > 0.9995) { + // Quaternions nearly identical β€” use linear blend + renormalize + s0 = 1 - t; s1 = t; + } else { + const angle = Math.acos(Math.min(dot, 1)); + const sinInv = 1 / Math.sin(angle); + s0 = Math.sin((1 - t) * angle) * sinInv; + s1 = Math.sin(t * angle) * sinInv; + } + + out[base + 3] = ax * s0 + bx * s1; + out[base + 4] = ay * s0 + by * s1; + out[base + 5] = az * s0 + bz * s1; + out[base + 6] = aw * s0 + bw * s1; + + // Normalize to handle floating-point drift + const len = Math.sqrt( + out[base + 3] ** 2 + out[base + 4] ** 2 + out[base + 5] ** 2 + out[base + 6] ** 2 + ); + if (len > 1e-6) { + const inv = 1 / len; + out[base + 3] *= inv; out[base + 4] *= inv; + out[base + 5] *= inv; out[base + 6] *= inv; + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// StreamingAnimationPlayer +// ───────────────────────────────────────────────────────────────────────────── + +export class StreamingAnimationPlayer { + /** + * @param {object} [opts] + * @param {number} [opts.targetDelayMs=60] β€” jitter buffer target delay in ms + * @param {number} [opts.maxBufferFrames=64] β€” max frames held before oldest are dropped + * @param {number} [opts.reconnectDelayMs=2000] β€” ms between auto-reconnect attempts + * @param {boolean} [opts.autoReconnect=true] + * @param {number} [opts.smoothingAlpha=0.15] β€” exponential smoothing for latency estimate + */ + constructor(opts = {}) { + this._targetDelayMs = opts.targetDelayMs ?? 60; + this._maxBufferFrames = opts.maxBufferFrames ?? BUFFER_MAX_FRAMES; + this._reconnectDelayMs = opts.reconnectDelayMs ?? 2000; + this._autoReconnect = opts.autoReconnect ?? true; + this._smoothingAlpha = opts.smoothingAlpha ?? 0.15; + + /** @type {Array<{serverTs:number, frameId:number, joints:Float32Array, facs:Float32Array, arrivalTs:number}>} */ + this._buffer = []; + this._lastFrame = null; // last consumed frame (for extrapolation) + this._events = {}; + + this._ws = null; + this._url = null; + this._streamType = STREAM_TYPE_WS; + this._connected = false; + this._reconnectTimer= null; + + // HTTP streaming + this._httpAbort = null; + + // Stats + this._totalFrames = 0; + this._droppedFrames= 0; + this._fpsCounter = 0; + this._fpsTs = performance.now(); + this._fps = 0; + this._smoothLatency= 0; + + // Playback start offset (localTs - targetDelay = playback clock) + this._startLocalTs = null; + this._startSrvTs = null; + } + + // ──────────────────────────────────────────────────────────────── connection + + /** + * Connect to a streaming endpoint. + * @param {string} url β€” ws:// wss:// or http:// https:// URL + * @param {'ws'|'http'} [type] β€” override transport; auto-detected from URL scheme if omitted + */ + connect(url, type) { + this.disconnect(); + + this._url = url; + this._streamType = type + ?? (url.startsWith('ws') ? STREAM_TYPE_WS : STREAM_TYPE_HTTP); + + if (this._streamType === STREAM_TYPE_WS) { + this._connectWS(url); + } else { + this._connectHTTP(url); + } + } + + disconnect() { + this._cancelReconnect(); + + if (this._ws) { + this._ws.onclose = null; + this._ws.onerror = null; + this._ws.onmessage = null; + this._ws.onopen = null; + if (this._ws.readyState < 2) this._ws.close(); + this._ws = null; + } + + if (this._httpAbort) { + this._httpAbort.abort(); + this._httpAbort = null; + } + + if (this._connected) { + this._connected = false; + this._emit('disconnect', { url: this._url }); + } + } + + // ──────────────────────────────────────────────────────────────── public API + + /** + * Return the interpolated/extrapolated pose for the given local timestamp. + * + * @param {number} [nowMs=performance.now()] + * @returns {{ + * joints: Float32Array|null, β€” jointCount Γ— 7 floats (tx,ty,tz, qx,qy,qz,qw) + * facs: Float32Array|null, β€” facsCount floats [0..1] + * frameId: number, + * age: number, β€” ms since this frame arrived + * interpolated: boolean + * }|null} null when no data has been received yet + */ + getInterpolatedPose(nowMs = performance.now()) { + if (this._buffer.length === 0 && !this._lastFrame) return null; + + // Compute playback position on the server timeline + let playTs; + if (this._startLocalTs !== null && this._startSrvTs !== null) { + const localElapsed = nowMs - this._startLocalTs; + playTs = this._startSrvTs + localElapsed - this._targetDelayMs; + } else if (this._lastFrame) { + return this._makeResult(this._lastFrame, false); + } else { + return null; + } + + // Find bracketing frames + const buf = this._buffer; + + if (buf.length === 0) { + // Extrapolate: no buffered frame β€” return last + return this._makeResult(this._lastFrame, false); + } + + // Find first frame at or after playTs + let hi = -1; + for (let i = 0; i < buf.length; i++) { + if (buf[i].serverTs >= playTs) { hi = i; break; } + } + + if (hi === -1) { + // All frames are in the past β€” consume the newest and extrapolate + const newest = buf[buf.length - 1]; + this._lastFrame = newest; + this._buffer = []; + return this._makeResult(newest, false); + } + + if (hi === 0) { + // Play position is before the oldest buffered frame + const oldest = buf[0]; + if (this._lastFrame) { + // Interpolate between lastFrame and oldest + const lo = this._lastFrame; + const span = oldest.serverTs - lo.serverTs; + const t = span > 0 ? Math.max(0, Math.min(1, (playTs - lo.serverTs) / span)) : 0; + return this._interpolate(lo, oldest, t); + } + return this._makeResult(oldest, false); + } + + // Normal case: interpolate between buf[hi-1] and buf[hi] + const lo = buf[hi - 1]; + const hi_ = buf[hi]; + const span = hi_.serverTs - lo.serverTs; + const t = span > 0 ? Math.max(0, Math.min(1, (playTs - lo.serverTs) / span)) : 0; + + // Evict frames that are now behind the playback pointer + this._lastFrame = lo; + this._buffer = buf.slice(hi - 1); // keep lo and everything after + + return this._interpolate(lo, hi_, t); + } + + /** + * Register an event listener. + * @param {'connect'|'disconnect'|'data'|'error'|'drop'|'reconnecting'} event + * @param {Function} cb + */ + on(event, cb) { + if (!this._events[event]) this._events[event] = []; + this._events[event].push(cb); + return this; + } + + /** Remove a previously registered event listener. */ + off(event, cb) { + if (!this._events[event]) return this; + this._events[event] = this._events[event].filter(fn => fn !== cb); + return this; + } + + /** Performance statistics snapshot. */ + get stats() { + return { + connected: this._connected, + bufferSize: this._buffer.length, + totalFrames: this._totalFrames, + dropped: this._droppedFrames, + fps: this._fps, + latencyMs: Math.round(this._smoothLatency), + targetDelayMs: this._targetDelayMs, + }; + } + + /** Set the jitter buffer target delay in milliseconds. */ + setTargetDelay(ms) { + this._targetDelayMs = Math.max(0, ms); + } + + /** + * Set the exponential smoothing factor for the latency estimate. + * @param {number} alpha β€” value in (0, 1]; higher = faster tracking, lower = smoother + */ + setSmoothingAlpha(alpha) { + this._smoothingAlpha = Math.max(0.001, Math.min(1, alpha)); + } + + /** + * Inject a pre-encoded binary frame directly into the jitter buffer. + * Useful for testing, mock data sources, and in-page simulations without + * a real network connection. + * @param {ArrayBuffer} buffer β€” frame encoded with encodeFrame() + */ + injectFrame(buffer) { + this._receiveFrame(buffer); + } + + destroy() { + this.disconnect(); + this._buffer = []; + this._lastFrame = null; + this._events = {}; + } + + // ──────────────────────────────────────────────────────────────── private + + /** @private */ + _connectWS(url) { + let ws; + try { + ws = new WebSocket(url); + } catch (e) { + this._emit('error', { message: e.message }); + this._scheduleReconnect(); + return; + } + ws.binaryType = 'arraybuffer'; + this._ws = ws; + + ws.onopen = () => { + this._connected = true; + this._resetPlaybackClock(); + this._emit('connect', { url, transport: 'ws' }); + }; + + ws.onmessage = (ev) => { + if (ev.data instanceof ArrayBuffer) { + this._receiveFrame(ev.data); + } + }; + + ws.onerror = (ev) => { + this._emit('error', { message: 'WebSocket error', event: ev }); + }; + + ws.onclose = (ev) => { + const wasConnected = this._connected; + this._connected = false; + this._ws = null; + if (wasConnected) this._emit('disconnect', { url, code: ev.code, reason: ev.reason }); + if (this._autoReconnect) this._scheduleReconnect(); + }; + } + + /** @private */ + _connectHTTP(url) { + const ctrl = new AbortController(); + this._httpAbort = ctrl; + + const doFetch = async () => { + try { + const res = await fetch(url, { + signal: ctrl.signal, + headers: { Accept: 'application/octet-stream' }, + }); + if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}`); + if (!res.body) throw new Error('No response body for HTTP stream'); + + this._connected = true; + this._resetPlaybackClock(); + this._emit('connect', { url, transport: 'http' }); + + const reader = res.body.getReader(); + // Buffer partial chunks (frames may be split across chunks) + let carry = new Uint8Array(0); + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + // Prepend any leftover bytes from previous chunk + let chunk; + if (carry.length > 0) { + chunk = new Uint8Array(carry.length + value.length); + chunk.set(carry); + chunk.set(value, carry.length); + carry = new Uint8Array(0); + } else { + chunk = value; + } + + // Parse frames out of the chunk + let pos = 0; + while (pos < chunk.length) { + if (chunk.length - pos < 12) { + // Not enough bytes for a header yet + carry = chunk.slice(pos); + break; + } + const view = new DataView(chunk.buffer, chunk.byteOffset + pos); + const jointCount = view.getUint8(8); + const facsCount = view.getUint8(9); + const frameSize = 12 + jointCount * 14 + facsCount * 2; + + if (chunk.length - pos < frameSize) { + // Incomplete frame β€” carry remainder to next chunk + carry = chunk.slice(pos); + break; + } + + const frameBuf = chunk.buffer.slice(chunk.byteOffset + pos, chunk.byteOffset + pos + frameSize); + this._receiveFrame(frameBuf); + pos += frameSize; + } + } + + // Stream ended + this._connected = false; + this._emit('disconnect', { url, reason: 'stream ended' }); + if (this._autoReconnect) this._scheduleReconnect(); + + } catch (err) { + if (err.name === 'AbortError') return; // intentional disconnect + this._connected = false; + this._emit('error', { message: err.message }); + if (this._autoReconnect) this._scheduleReconnect(); + } + }; + + doFetch(); + } + + /** @private Parse and buffer one binary frame. */ + _receiveFrame(buffer) { + let frame; + try { + frame = decodeFrame(buffer); + } catch (e) { + this._emit('error', { message: e.message }); + return; + } + + // Sync playback clock on first frame + if (this._startLocalTs === null) { + this._startLocalTs = frame.arrivalTs; + this._startSrvTs = frame.serverTs; + } + + // Update smoothed latency estimate: + // latency β‰ˆ arrival - targetDelay - serverTs (mapped to local clock) + const estimatedLatency = frame.arrivalTs - this._startLocalTs + - (frame.serverTs - this._startSrvTs); + this._smoothLatency = this._smoothLatency * (1 - this._smoothingAlpha) + + Math.max(0, estimatedLatency) * this._smoothingAlpha; + + // Evict oldest frames if buffer is full + if (this._buffer.length >= this._maxBufferFrames) { + this._droppedFrames++; + this._buffer.shift(); + this._emit('drop', { frameId: frame.frameId }); + } + + // Insert in order (normally frames arrive in order β€” fast path) + if (this._buffer.length === 0 || frame.serverTs >= this._buffer[this._buffer.length - 1].serverTs) { + this._buffer.push(frame); + } else { + // Out-of-order arrival β€” insert sorted by serverTs + let i = this._buffer.length - 1; + while (i > 0 && this._buffer[i - 1].serverTs > frame.serverTs) i--; + this._buffer.splice(i, 0, frame); + } + + // FPS counter + this._totalFrames++; + this._fpsCounter++; + const now = performance.now(); + if (now - this._fpsTs >= 1000) { + this._fps = (this._fpsCounter * 1000) / (now - this._fpsTs); + this._fpsCounter = 0; + this._fpsTs = now; + } + + this._emit('data', { frameId: frame.frameId, serverTs: frame.serverTs }); + } + + /** @private */ + _interpolate(frameA, frameB, t) { + const jA = frameA.joints, jB = frameB.joints; + const jointCount = Math.min(jA.length, jB.length) / 7 | 0; + const out = new Float32Array(jointCount * 7); + + for (let j = 0; j < jointCount; j++) { + const base = j * 7; + // Lerp position + out[base] = jA[base] + (jB[base] - jA[base]) * t; + out[base + 1] = jA[base + 1] + (jB[base + 1] - jA[base + 1]) * t; + out[base + 2] = jA[base + 2] + (jB[base + 2] - jA[base + 2]) * t; + // Slerp rotation + slerpJointQuat(jA, jB, t, out, base); + } + + const fA = frameA.facs, fB = frameB.facs; + const facsCount = Math.min(fA.length, fB.length); + const facs = new Float32Array(facsCount); + for (let f = 0; f < facsCount; f++) { + facs[f] = fA[f] + (fB[f] - fA[f]) * t; + } + + return { + joints: out, + facs, + frameId: frameB.frameId, + age: performance.now() - frameB.arrivalTs, + interpolated: true, + }; + } + + /** @private */ + _makeResult(frame, interpolated) { + return { + joints: frame.joints, + facs: frame.facs, + frameId: frame.frameId, + age: performance.now() - frame.arrivalTs, + interpolated, + }; + } + + /** @private */ + _resetPlaybackClock() { + this._startLocalTs = null; + this._startSrvTs = null; + this._buffer = []; + this._smoothLatency = 0; + this._fpsCounter = 0; + this._fpsTs = performance.now(); + } + + /** @private */ + _scheduleReconnect() { + this._cancelReconnect(); + this._emit('reconnecting', { url: this._url, delayMs: this._reconnectDelayMs }); + this._reconnectTimer = setTimeout(() => { + this._reconnectTimer = null; + if (this._url) this.connect(this._url, this._streamType); + }, this._reconnectDelayMs); + } + + /** @private */ + _cancelReconnect() { + if (this._reconnectTimer !== null) { + clearTimeout(this._reconnectTimer); + this._reconnectTimer = null; + } + } + + /** @private */ + _emit(event, data) { + const cbs = this._events[event]; + if (!cbs) return; + for (const cb of cbs) { + try { cb(data); } catch (e) { console.error('[StreamingAnimationPlayer] event handler error:', e); } + } + } +} diff --git a/src/sdk/OpenHuman.js b/src/sdk/OpenHuman.js index c44e1c5..c15e051 100644 --- a/src/sdk/OpenHuman.js +++ b/src/sdk/OpenHuman.js @@ -16,6 +16,7 @@ import { Character } from '../scene/Character.js'; import { Node } from '../scene/Node.js'; import { Vec3 } from '../math/Vec3.js'; import { MorphController, FACS_NAMES } from '../animation/MorphController.js'; +import { StreamingAnimationPlayer } from '../animation/StreamingAnimationPlayer.js'; // Module-level scratch Vec3 for SDK lookAt calls β€” avoids per-call allocation. const _lookAtScratch = new Vec3(); @@ -159,11 +160,126 @@ class OpenHumanInstance { getWeight(name) { return character._morphController?.getWeight(name) ?? 0; }, }; - this.streaming = { - connect(url) { /* TODO: WebSocket / WebRTC data channel */ }, - disconnect() { /* TODO */ }, - onData(cb) { /* TODO */ }, - }; + this.streaming = (() => { + /** @type {StreamingAnimationPlayer|null} */ + let player = null; + + const api = { + /** + * Connect to a streaming endpoint (WebSocket or HTTP). + * @param {string} url β€” ws:// / wss:// or http:// / https:// + * @param {'ws'|'http'} [type] β€” transport override (auto-detected when omitted) + * @param {object} [opts] β€” StreamingAnimationPlayer constructor options + */ + connect(url, type, opts) { + if (!player) { + player = new StreamingAnimationPlayer(opts); + + player.on('connect', d => self._emit('streaming:connect', d)); + player.on('disconnect', d => self._emit('streaming:disconnect', d)); + player.on('error', d => self._emit('streaming:error', d)); + player.on('drop', d => self._emit('streaming:drop', d)); + player.on('reconnecting', d => self._emit('streaming:reconnecting', d)); + player.on('data', d => { + self._emit('streaming:data', d); + }); + } + player.connect(url, type); + }, + + /** Disconnect from the current stream. */ + disconnect() { + player?.disconnect(); + }, + + /** + * Register a callback for raw incoming frame events. + * @param {Function} cb β€” receives { frameId, serverTs } + */ + onData(cb) { + if (!player) player = new StreamingAnimationPlayer(); + player.on('data', cb); + }, + + /** + * Get the interpolated pose for the current moment. + * Applies the result directly to the character's morph controller + * and skeleton (if available) when applyToCharacter is true. + * + * @param {number} [nowMs=performance.now()] + * @param {boolean} [applyToCharacter=true] + * @returns {{ joints: Float32Array|null, facs: Float32Array|null, frameId: number, age: number }|null} + */ + getInterpolatedPose(nowMs, applyToCharacter = true) { + if (!player) return null; + const pose = player.getInterpolatedPose(nowMs); + if (!pose) return null; + + if (applyToCharacter) { + // Apply FACS weights to the morph controller + if (pose.facs && character._morphController) { + const mc = character._morphController; + for (let i = 0; i < pose.facs.length && i < mc.numMorphs; i++) { + mc.setByIndex(i, pose.facs[i]); + } + } + + // Apply joint transforms to the skeleton if available + if (pose.joints && character._skeleton) { + const skel = character._skeleton; + const joints = skel.joints ?? []; + const n = Math.min(pose.joints.length / 7 | 0, joints.length); + for (let j = 0; j < n; j++) { + const base = j * 7; + const joint = joints[j]; + if (!joint) continue; + if (joint.node) { + joint.node.position.set( + pose.joints[base], + pose.joints[base + 1], + pose.joints[base + 2] + ); + // quaternion as x,y,z,w + joint.node.rotation.set( + pose.joints[base + 3], + pose.joints[base + 4], + pose.joints[base + 5], + pose.joints[base + 6] + ); + } + } + } + } + + return pose; + }, + + /** Performance stats from the underlying jitter buffer player. */ + get stats() { + return player?.stats ?? null; + }, + + /** + * Set jitter buffer target delay in milliseconds (default 60 ms). + * Lower = less latency but more susceptibility to packet-loss pops. + * @param {number} ms + */ + setTargetDelay(ms) { + player?.setTargetDelay(ms); + }, + + /** Destroy the underlying player and free resources. */ + destroy() { + player?.destroy(); + player = null; + }, + + /** Expose the raw player instance for advanced use. */ + get player() { return player; }, + }; + + return api; + })(); this.camera = { setPosition(x, y, z) { camera.setPosition(x, y, z); }, @@ -377,3 +493,4 @@ export { AnimationClip, Pose } from '../animation/AnimationClip.js'; export { AnimationGraph } from '../animation/AnimationGraph.js'; export { GPUSkinning } from '../animation/GPUSkinning.js'; export { MorphController, FACS_NAMES, MAX_MORPH_TARGETS } from '../animation/MorphController.js'; +export { StreamingAnimationPlayer, encodeFrame, STREAM_TYPE_WS, STREAM_TYPE_HTTP } from '../animation/StreamingAnimationPlayer.js';