Skip to content

fix: smooth agent visual audio response #594

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rogerchappel merged 1 commit into from
May 24, 2026
Merged

fix: smooth agent visual audio response #594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 85 additions & 23 deletions src/components/chat/agent-visualizer.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,15 @@ interface DrawFrame {
intensity: number;
reducedMotion: boolean;
time: number;
primaryPhase: number;
secondaryPhase: number;
tertiaryPhase: number;
width: number;
height: number;
}

type DrawFrameInput = Omit<DrawFrame, "time" | "width" | "height" | "reducedMotion" | "primaryPhase" | "secondaryPhase" | "tertiaryPhase">;

const INTENSITY_SCALE = {
calm: 0.78,
balanced: 1,
Expand All @@ -73,9 +78,12 @@ const STYLE_LABEL: Record<VisualStyleId, string> = {

const clamp = (value: number, min = 0, max = 1) => Math.max(min, Math.min(max, value));

function parseRgb(value: string): Rgb {
const channels = value
.split(",")
function parseRgb(value: string, element?: HTMLElement | null): Rgb {
const resolved = value.startsWith("var(") && element
? getComputedStyle(element).getPropertyValue(value.slice(4, -1).trim())
: value;
const channels = resolved
.split(/[,\s]+/)
.map((channel) => Number.parseInt(channel.trim(), 10))
.filter((channel) => Number.isFinite(channel));
if (channels.length < 3) return [99, 183, 170];
Expand Down Expand Up @@ -104,6 +112,11 @@ function stateEnergy(frame: DrawFrame) {
return 0.18 + frame.motionLevel * 0.22;
}

function smoothLevel(current: number, target: number, delta: number) {
const rate = target > current ? 18 : 7.5;
return current + (target - current) * (1 - Math.exp(-rate * delta));
}

function activeColor(frame: DrawFrame) {
if (frame.state === "listening") return frame.listening;
if (frame.state === "speaking") return frame.speaking;
Expand Down Expand Up @@ -216,12 +229,11 @@ function drawBackground(ctx: CanvasRenderingContext2D, frame: DrawFrame, base: R
}

function drawOrbitalReactor(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
const { width, height, time } = frame;
const { width, height } = frame;
const cx = width / 2;
const cy = height / 2;
const size = Math.min(width, height);
const energy = stateEnergy(frame);
const motion = frame.reducedMotion ? 0 : frame.motionLevel * frame.intensity;
const accent = activeColor(frame);
const graphite: Rgb = [38, 43, 48];
const pearl: Rgb = [236, 241, 240];
Expand All @@ -230,23 +242,23 @@ function drawOrbitalReactor(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
drawBackground(ctx, frame, graphite, true);
glow(ctx, cx, cy, r * 1.4, accent, frame.state === "muted" ? 0.04 : 0.13 + energy * 0.1);

const rotation = time * (0.18 + motion * 0.55);
const rotation = frame.primaryPhase;
for (let i = 0; i < 5; i += 1) {
const radius = r * (0.48 + i * 0.16);
const alpha = 0.11 + i * 0.035;
circle(ctx, cx, cy, radius, rgba(i % 2 ? pearl : graphite, alpha), i === 3 ? 1.6 : 1);
}

for (let i = 0; i < 9; i += 1) {
const offset = rotation + i * 0.72;
const offset = rotation * (i % 2 ? -0.62 : 1) + i * 0.72;
const radius = r * (0.57 + (i % 3) * 0.18);
const length = 0.22 + energy * 0.28 + (i % 2) * 0.08;
const stroke = i % 3 === 0 ? accent : mix(accent, pearl, 0.42);
arc(ctx, cx, cy, radius, offset, offset + length, rgba(stroke, 0.22 + energy * 0.48), 2.2 + energy * 2.2);
}

for (let i = 0; i < 24; i += 1) {
const angle = rotation * 0.7 + (i / 24) * Math.PI * 2;
const angle = frame.secondaryPhase + (i / 24) * Math.PI * 2;
const radius = r * (0.8 + (i % 4) * 0.045);
const x = cx + Math.cos(angle) * radius;
const y = cy + Math.sin(angle) * radius;
Expand All @@ -263,6 +275,27 @@ function drawOrbitalReactor(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
ctx.arc(cx, cy, r * (0.19 + energy * 0.035), 0, Math.PI * 2);
ctx.fill();
circle(ctx, cx, cy, r * (0.21 + energy * 0.03), rgba(pearl, 0.42), 1.7);
if (frame.state === "speaking") {
for (let i = 0; i < 3; i += 1) {
const wave = (frame.tertiaryPhase * 0.45 + i / 3) % 1;
circle(ctx, cx, cy, r * (0.32 + wave * 0.82), rgba(frame.speaking, (1 - wave) * 0.28), 1.8 + energy * 1.8);
}
} else if (frame.state === "listening") {
for (let i = 0; i < 18; i += 1) {
const angle = -frame.tertiaryPhase * 0.8 + (i / 18) * Math.PI * 2;
const inner = r * (0.2 + frame.visualVolume * 0.08);
const outer = r * (0.38 + frame.visualVolume * 0.18);
line(
ctx,
cx + Math.cos(angle) * inner,
cy + Math.sin(angle) * inner,
cx + Math.cos(angle) * outer,
cy + Math.sin(angle) * outer,
rgba(frame.listening, 0.1 + frame.visualVolume * 0.22),
1,
);
}
}
}

function drawNeuralConstellation(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
Expand Down Expand Up @@ -306,15 +339,15 @@ function drawNeuralConstellation(ctx: CanvasRenderingContext2D, frame: DrawFrame
const distance = Math.hypot(dx, dy);
const threshold = size * (frame.compact ? 0.19 : 0.16);
if (distance > threshold) continue;
const route = frame.state === "processing" ? Math.sin(time * 4 + i * 0.8 + j) * 0.5 + 0.5 : 0.25;
const route = frame.state === "processing" ? Math.sin(frame.primaryPhase * 2.6 + i * 0.8 + j) * 0.5 + 0.5 : 0.25;
const alpha = (1 - distance / threshold) * (0.15 + energy * 0.24 + route * 0.18);
line(ctx, a.x, a.y, b.x, b.y, rgba(route > 0.7 ? accent : cool, alpha), 1 + route * 1.4);
}
}

for (let i = 0; i < points.length; i += 1) {
const point = points[i];
const wave = Math.max(0, Math.sin(time * (2.2 + motion * 2.8) - point.rank * 0.75));
const wave = Math.max(0, Math.sin(frame.secondaryPhase * (1.4 + motion) - point.rank * 0.75));
const active = frame.state === "listening" ? wave : frame.state === "speaking" ? 1 - wave * 0.45 : frame.state === "processing" ? wave * 0.9 : 0.28;
const radius = (frame.compact ? 2.1 : 3.1) + active * energy * (frame.immersive ? 5.8 : 3.8);
glow(ctx, point.x, point.y, radius * 4.2, accent, active * 0.08);
Expand All @@ -326,12 +359,11 @@ function drawNeuralConstellation(ctx: CanvasRenderingContext2D, frame: DrawFrame
}

function drawHologramWaveform(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
const { width, height, time } = frame;
const { width, height } = frame;
const cx = width / 2;
const cy = height / 2;
const size = Math.min(width, height);
const energy = stateEnergy(frame);
const motion = frame.reducedMotion ? 0 : frame.motionLevel * frame.intensity;
const accent = activeColor(frame);
const cyan: Rgb = [62, 205, 226];
const violet: Rgb = [102, 116, 210];
Expand All @@ -351,7 +383,7 @@ function drawHologramWaveform(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
frame.state === "muted"
? size * 0.008
: size * (0.018 + energy * (frame.state === "speaking" ? 0.085 : 0.062)) * (1 - depth * 0.18);
const phase = time * (1.3 + motion * 2.5 + depth) + depth * 4;
const phase = frame.primaryPhase * (1.1 + depth * 0.45) + depth * 4;
const layerColor = mix(layer % 2 ? accent : cyan, violet, depth * 0.55);
const left = cx - bandWidth / 2;
const right = cx + bandWidth / 2;
Expand All @@ -363,7 +395,7 @@ function drawHologramWaveform(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
const envelope = Math.sin(progress * Math.PI);
const carrier = Math.sin(progress * Math.PI * (4 + layer) + phase);
const detail = Math.sin(progress * Math.PI * (11 + layer * 2) - phase * 0.74);
const compression = frame.state === "processing" ? Math.sign(Math.sin(progress * Math.PI * 18 + time * 7)) * 0.38 : 1;
const compression = frame.state === "processing" ? Math.sign(Math.sin(progress * Math.PI * 18 + frame.secondaryPhase * 4)) * 0.38 : 1;
const yy = y + (carrier * 0.74 + detail * 0.26) * amplitude * envelope * compression;
if (step === 0) ctx.moveTo(x, yy);
else ctx.lineTo(x, yy);
Expand All @@ -378,7 +410,7 @@ function drawHologramWaveform(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
line(ctx, left, y + size * 0.07, right, y + size * 0.07, rgba(pearl, 0.04 + (1 - depth) * 0.04), 1);
}

const scanX = cx - bandWidth / 2 + ((time * (0.16 + motion * 0.28)) % 1) * bandWidth;
const scanX = cx - bandWidth / 2 + ((frame.tertiaryPhase * 0.12) % 1) * bandWidth;
const scan = ctx.createLinearGradient(scanX - 18, cy, scanX + 18, cy);
scan.addColorStop(0, rgba(cyan, 0));
scan.addColorStop(0.48, rgba(pearl, frame.state === "processing" ? 0.38 : 0.18));
Expand All @@ -389,18 +421,17 @@ function drawHologramWaveform(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
}

function drawCommandCore(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
const { width, height, time } = frame;
const { width, height } = frame;
const cx = width / 2;
const cy = height / 2;
const size = Math.min(width, height);
const energy = stateEnergy(frame);
const motion = frame.reducedMotion ? 0 : frame.motionLevel * frame.intensity;
const accent = activeColor(frame);
const amber: Rgb = [228, 174, 84];
const steel: Rgb = [30, 38, 48];
const chalk: Rgb = [235, 236, 228];
const r = size * (frame.immersive ? 0.35 : 0.33);
const sweep = time * (0.55 + motion * 1.8);
const sweep = frame.primaryPhase;

drawBackground(ctx, frame, steel, true);

Expand All @@ -414,7 +445,7 @@ function drawCommandCore(ctx: CanvasRenderingContext2D, frame: DrawFrame) {
const hot = frame.state === "listening"
? i / 28 < frame.visualVolume
: frame.state === "processing"
? Math.sin(time * 8 + i * 0.6) > 0.35
? Math.sin(frame.secondaryPhase * 4 + i * 0.6) > 0.35
: frame.state === "speaking"
? Math.sin(angle - sweep) > 0.76
: false;
Expand Down Expand Up @@ -468,18 +499,24 @@ const DRAWERS: Record<VisualStyleId, (ctx: CanvasRenderingContext2D, frame: Draw
function AgentIdentityCanvas({
styleId,
frame,
colorSources,
}: {
styleId: VisualStyleId;
frame: Omit<DrawFrame, "time" | "width" | "height" | "reducedMotion">;
frame: DrawFrameInput;
colorSources: { listening: string; speaking: string; processing: string };
}) {
const canvasRef = useRef<HTMLCanvasElement | null>(null);
const frameRef = useRef(frame);
const styleRef = useRef(styleId);
const colorSourcesRef = useRef(colorSources);
const smoothedRef = useRef<DrawFrameInput | null>(null);
const phaseRef = useRef({ primary: 0, secondary: 0, tertiary: 0, lastNow: 0 });

useEffect(() => {
frameRef.current = frame;
styleRef.current = styleId;
}, [frame, styleId]);
colorSourcesRef.current = colorSources;
}, [colorSources, frame, styleId]);

useEffect(() => {
const canvas = canvasRef.current;
Expand All @@ -504,11 +541,32 @@ function AgentIdentityCanvas({
const { width, height } = setupCanvas(canvas, ctx);
const time = reducedMotion ? 0 : (now - start) / 1000;
if (reducedMotion) start = now;
const delta = phaseRef.current.lastNow ? Math.min(0.05, (now - phaseRef.current.lastNow) / 1000) : 1 / 60;
phaseRef.current.lastNow = now;
const target = frameRef.current;
const previous = smoothedRef.current ?? target;
const sources = colorSourcesRef.current;
const smoothed: DrawFrameInput = {
...target,
listening: parseRgb(sources.listening, canvas),
speaking: parseRgb(sources.speaking, canvas),
processing: parseRgb(sources.processing, canvas),
visualVolume: smoothLevel(previous.visualVolume, target.visualVolume, delta),
motionLevel: smoothLevel(previous.motionLevel, target.motionLevel, delta),
};
smoothedRef.current = smoothed;
const speed = reducedMotion ? 0 : 0.38 + smoothed.motionLevel * 1.45 + smoothed.visualVolume * 0.36;
phaseRef.current.primary += speed * delta;
phaseRef.current.secondary += (speed * 0.63 + 0.18) * delta;
phaseRef.current.tertiary += (speed * 1.22 + 0.24) * delta;
clear(ctx, width, height);
const nextFrame = {
...frameRef.current,
...smoothed,
reducedMotion,
time,
primaryPhase: phaseRef.current.primary,
secondaryPhase: phaseRef.current.secondary,
tertiaryPhase: phaseRef.current.tertiary,
width,
height,
};
Expand Down Expand Up @@ -575,7 +633,11 @@ export function AgentVisualizer({
className={`relative flex max-w-full items-center justify-center overflow-visible rounded-full select-none outline-none transition-transform duration-300 hover:scale-[1.01] focus-visible:ring-2 focus-visible:ring-offset-2 focus-visible:ring-offset-[var(--bg-primary)] ${sizeClass}`}
style={{ "--tw-ring-color": rgba(accent, 0.5) } as CSSProperties}
>
<AgentIdentityCanvas styleId={styleId} frame={frame} />
<AgentIdentityCanvas
styleId={styleId}
frame={frame}
colorSources={{ listening: listeningRgb, speaking: speakingRgb, processing: processingRgb }}
/>
</Wrapper>
);
}
4 changes: 2 additions & 2 deletions src/components/chat/voice-agent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,8 @@ export function VoiceAgent({
const normalized = Math.max(0, Math.min(1, Number.isFinite(nextLevel) ? nextLevel : 0));
const now = Date.now();
if (
now - lastVolumeLevelAtRef.current < 50 &&
Math.abs(normalized - lastVolumeLevelRef.current) < 0.035
now - lastVolumeLevelAtRef.current < 24 &&
Math.abs(normalized - lastVolumeLevelRef.current) < 0.012
) {
return;
}
Expand Down
Loading