feat(voice): an extension can draw the avatar in voice mode, chosen per agent

core/server/src/orb-style.ts

@@ -86,6 +86,13 @@ export const ORB_PALETTES: Record<string, OrbColors> = {
 };
 
 export interface OrbStyle {
+  /**
+   * What voice mode draws for the agent: "orb", the portal's own, drawn with
+   * the rest of this, or an avatar an extension offers, by its entry's
+   * "<package>/<id>" (the `voice.avatar` slot). The page draws the orb for an
+   * avatar it does not have; the server cannot know which a page has.
+   */
+  avatar: string;
   personality: OrbPersonality;
   /** The palette the colors came from, or "custom" once one was changed by hand. */
   palette: string;
@@ -105,6 +112,7 @@ export interface OrbStyle {
 }
 
 export const DEFAULT_ORB: OrbStyle = {
+  avatar: "orb",
   personality: "balanced",
   palette: "aurora",
   colors: { ...ORB_PALETTES.aurora },
@@ -122,6 +130,8 @@ export const DEFAULT_ORB: OrbStyle = {
 };
 
 const HEX = /^#[0-9a-f]{6}$/i;
+/** The longest "<package>/<id>" kept for an extension's avatar. */
+const AVATAR_KEY_MAX = 256;
 const STATES: OrbState[] = ["idle", "input", "output", "muted"];
 
 /** A #rrggbb colour, lower-cased; anything else is the default. */
@@ -146,6 +156,7 @@ export function normalizeOrb(value: unknown): OrbStyle {
   const colors = Object.fromEntries(STATES.map((s) => [s, hex(given[s], DEFAULT_ORB.colors[s])])) as OrbColors;
   const palette = typeof v.palette === "string" && (Object.hasOwn(ORB_PALETTES, v.palette) || v.palette === "custom") ? v.palette : DEFAULT_ORB.palette;
   return {
+    avatar: typeof v.avatar === "string" && v.avatar && v.avatar.length <= AVATAR_KEY_MAX ? v.avatar : DEFAULT_ORB.avatar,
     personality,
     palette,
     colors,

core/server/src/server.ts

@@ -971,6 +971,17 @@ app.get("/api/agent/orb", (req, res) => {
   res.json(orbOf(agentFor(session?.workspace)));
 });
 
+/**
+ * The agent a chat (`?session=`) is with and its avatar, which voice mode
+ * shows: the agent whose home the chat is in, the one a chat in a project is
+ * linked to, and else the first.
+ */
+app.get("/api/agent/avatar", (req, res) => {
+  const session = typeof req.query.session === "string" ? getSession(req.query.session) : undefined;
+  const agent = agentFor(session?.workspace);
+  res.json({ agent: { id: agent.id, name: agent.name }, orb: orbOf(agent) });
+});
+
 app.post("/api/sessions", (req, res) => {
   const { workspace, agent: agentId } = req.body ?? {};
   const title = cleanTitle(req.body?.title);

core/server/test/agents.test.mjs

@@ -212,6 +212,28 @@ test("each agent has an avatar of its own, and voice mode shows its chat's agent
   assert.equal((await call("PUT", "/api/agents/ops/orb", [])).status, 400);
 });
 
+test("an agent's avatar can be one an extension offers, and voice mode is told the chat's agent with it", async () => {
+  const scout = (await call("POST", "/api/agents", { name: "Scout" })).body;
+  assert.equal(scout.orb.avatar, "orb");
+  const saved = await call("PUT", "/api/agents/scout/orb", { ...scout.orb, avatar: "test-puppet/puppet", hat: "crown" });
+  assert.equal(saved.status, 200);
+  assert.equal(saved.body.avatar, "test-puppet/puppet");
+  const agents = (await call("GET", "/api/agents")).body.agents;
+  assert.equal(agents.find((a) => a.id === "scout").orb.avatar, "test-puppet/puppet");
+  const first = agents.find((a) => a.first);
+
+  const chat = (await call("POST", "/api/sessions", { agent: "scout" })).body;
+  const avatar = (await call("GET", `/api/agent/avatar?session=${chat.id}`)).body;
+  assert.deepEqual(avatar.agent, { id: "scout", name: "Scout" });
+  // The orb's look is kept with the choice: it is what is drawn where the extension's cannot be.
+  assert.deepEqual([avatar.orb.avatar, avatar.orb.hat], ["test-puppet/puppet", "crown"]);
+  const homeChat = (await call("POST", "/api/sessions", {})).body;
+  assert.deepEqual((await call("GET", `/api/agent/avatar?session=${homeChat.id}`)).body, { agent: { id: first.id, name: first.name }, orb: first.orb });
+  // Saved without it, the orb is chosen again, as any field left out is its default.
+  const { avatar: _, ...look } = saved.body;
+  assert.equal((await call("PUT", "/api/agents/scout/orb", look)).body.avatar, "orb");
+});
+
 test("an agent speaks with a voice its voice mode's source offers, or the source's own choice", async () => {
   await call("POST", "/api/agents", { name: "Herald" });
   assert.equal((await call("GET", "/api/agents")).body.agents.find((a) => a.id === "herald").voice, "");

core/server/test/project-agent-api.test.mjs

@@ -143,4 +143,8 @@ test("the avatar of a chat in a linked project is its agent's", async () => {
   // A chat in a project that works with no one is the first agent's, as it was.
   const plain = await json("/api/sessions", "POST", { workspace: "dev" });
   assert.notEqual((await json(`/api/agent/orb?session=${plain.id}`)).personality, "calm");
+  // And so is the agent voice mode is told it is with.
+  const avatar = await json(`/api/agent/avatar?session=${linked.id}`);
+  assert.deepEqual([avatar.agent.id, avatar.agent.name, avatar.orb.personality], [nova, "Nova", "calm"]);
+  assert.notEqual((await json(`/api/agent/avatar?session=${plain.id}`)).agent.id, nova);
 });

core/web/src/api.ts

@@ -972,8 +972,8 @@ export const api = {
   agentSkill: (agent: string, skill: string) =>
     json<{ content: string }>(`/api/agents/${encodeURIComponent(agent)}/skills/${encodeURIComponent(skill)}`),
   agentSetup: (agent: string) => json<AgentSetup>(`/api/agents/${encodeURIComponent(agent)}/setup`),
-  /** The avatar voice mode shows for a chat: its agent's. */
-  chatOrb: (session: string) => json<OrbStyle>(`/api/agent/orb?session=${encodeURIComponent(session)}`),
+  /** The agent a chat is with, and the avatar voice mode shows for it. */
+  chatAvatar: (session: string) => json<{ agent: { id: string; name: string }; orb: OrbStyle }>(`/api/agent/avatar?session=${encodeURIComponent(session)}`),
   /** An agent's avatar: the voice-mode orb's look and personality. */
   setAgentOrb: (agent: string, style: OrbStyle) =>
     json<OrbStyle>(`/api/agents/${encodeURIComponent(agent)}/orb`, { method: "PUT", body: JSON.stringify(style) }),

core/web/src/components/AgentAvatar.tsx

@@ -0,0 +1,57 @@
+import { Suspense, useEffect, useMemo, useState, type MutableRefObject } from "react";
+import { api } from "../api";
+import { failedIn } from "../extensions/loader";
+import { useSlot } from "../extensions/slots";
+import { DEFAULT_ORB, type OrbState, type OrbStyle } from "../../../server/src/orb-style";
+import { ErrorBoundary } from "./ErrorBoundary";
+import { ORB_STYLE_EVENT, VoiceOrb, type VoiceLevels } from "./VoiceOrb";
+
+/**
+ * The agent a chat talks to, and its avatar: the default orb and no agent until
+ * the server answers, then whatever is saved. Asked again when an avatar is
+ * saved, since it can be this one.
+ */
+export function useChatAvatar(session: string): { agent?: { id: string; name: string }; look: OrbStyle } {
+  const [avatar, setAvatar] = useState<{ agent?: { id: string; name: string }; look: OrbStyle }>({ look: DEFAULT_ORB });
+  useEffect(() => {
+    let live = true;
+    const load = () => api.chatAvatar(session).then(({ agent, orb }) => { if (live) setAvatar({ agent, look: orb }); }).catch(() => {});
+    void load();
+    window.addEventListener(ORB_STYLE_EVENT, load);
+    return () => { live = false; window.removeEventListener(ORB_STYLE_EVENT, load); };
+  }, [session]);
+  return avatar;
+}
+
+/**
+ * An agent's avatar, wherever the portal shows one: the orb, or what an
+ * extension draws in its place (`voice.avatar`) for an agent whose look chose
+ * it (`look.avatar`, "<package>/<id>"). The orb is drawn while that entry is
+ * not in the page (its extension is not installed, is switched off, has not
+ * loaded or not registered it yet), and in place of it when it fails to draw,
+ * which Settings → Extensions then says. `thinking` is the stage's (see
+ * AvatarMode); `still` is for a picture of the agent rather than voice mode.
+ *
+ * The orb and the agent are made once for what they show: the stage is drawn
+ * again with every token of a reply, and an extension's part is drawn again
+ * only when what it is given changes.
+ */
+export function AgentAvatar({ look, agent, mode, thinking = false, levels, still = false }: {
+  look: OrbStyle; agent?: { id: string; name: string }; mode: OrbState; thinking?: boolean; levels: MutableRefObject<VoiceLevels>; still?: boolean;
+}) {
+  const orb = useMemo(() => <VoiceOrb mode={mode} levels={levels} look={look} />, [mode, levels, look]);
+  const id = agent?.id, name = agent?.name;
+  const who = useMemo(() => (id === undefined || name === undefined ? undefined : { id, name }), [id, name]);
+  const entry = useSlot("voice.avatar").find((e) => e.key === look.avatar);
+  if (!entry || !who) return orb;
+  return (
+    <div className="voice-avatar-x" aria-hidden="true">
+      <ErrorBoundary resetKey={`${entry.key}:${who.id}`} fallback={() => orb} onError={failedIn(entry.extension)}>
+        {/* Its own: a part still being fetched would otherwise blank the page above it until it is. */}
+        <Suspense fallback={orb}>
+          <entry.component agent={who} mode={thinking ? "thinking" : mode} levels={levels} still={still} orb={orb} />
+        </Suspense>
+      </ErrorBoundary>
+    </div>
+  );
+}

core/web/src/components/AgentPage.tsx

@@ -3,7 +3,8 @@ import { useSearchParams } from "react-router-dom";
 import { OrbStudio } from "./OrbStudio";
 import { ActivityFeed, HeartbeatSettings } from "./AgentHeartbeat";
 import { AgentSkills } from "./AgentSkills";
-import { VoiceOrb, type VoiceLevels } from "./VoiceOrb";
+import type { VoiceLevels } from "./VoiceOrb";
+import { AgentAvatar } from "./AgentAvatar";
 import {
   LuBot,
   LuCheck,
@@ -163,10 +164,12 @@ function AgentCards({ agents, onOpen, onNew }: { agents: Agent[]; onOpen: (id: s
                 onClick={() => onOpen(a.id)}
                 className="group relative flex w-full flex-col items-center rounded-2xl border border-line bg-raised/40 px-3 pb-4 pt-5 text-center transition hover:border-accent/40 hover:bg-raised/70"
               >
-                {/* Room around the orb for its glow, and for a hat or a prop. */}
-                <div className="flex h-28 w-28 items-center justify-center rounded-2xl bg-[#0b1220]">
-                  <div className="voice-avatar w-[56%]">
-                    <VoiceOrb mode="idle" levels={still} look={a.orb} />
+                {/* Room around the orb for its glow, and for a hat or a prop. It is all the avatar's box. An
+                    extension's avatar, which may draw past its box on the stage, is cut at its edge here, where
+                    that would reach over the agent's name; so is the orb it falls back to, hat and all. */}
+                <div className="flex h-28 w-28 items-center justify-center rounded-2xl bg-[#0b1220] has-[.voice-avatar-x]:overflow-hidden">
+                  <div className="voice-avatar">
+                    <AgentAvatar look={a.orb} agent={a} mode="idle" levels={still} still />
                   </div>
                 </div>
                 {a.unread > 0 && (
@@ -386,7 +389,7 @@ function AgentView({
           {back}
           <PageHeader
             icon={<LuBot />}
-            media={<OrbStudio agent={agent.id} orb={agent.orb} voice={agent.voice} onSaved={() => void onChanged()} />}
+            media={<OrbStudio agent={agent.id} name={agent.name} orb={agent.orb} voice={agent.voice} onSaved={() => void onChanged()} />}
             title={
               renaming === "agent" ? (
                 <TitleInput value={agent.name} label={t("Agent name")} className="w-full" onCommit={renameAgent} onCancel={() => setRenaming(null)} />