Sync from internal repo (2026/07/22)

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## [4.36.4]
+
+- Add `aac` to the streaming `encoding` options — accepts an AAC stream in ADTS framing. Like `opus`/`ogg_opus`, AAC is self-describing, so `sampleRate` is optional for it (it remains required for PCM encodings and for dual-channel mode)
+- Add opt-in `sessionHeartbeat` streaming param — when enabled, the server emits a periodic `Heartbeat` message, surfaced via the new `heartbeat` event with `total_audio_received_ms`, `total_duration_ms`, `realtime_factor`, and `max_speech_probability`. Requires the `universal-3-5-pro` streaming model
+
 ## [4.36.3]
 
 - Target the canonical `/v1` sync API routes (`/v1/transcribe`, `/v1/warm`); the unprefixed paths remain served for older SDK versions

package.json

@@ -1,6 +1,6 @@
 {
   "name": "assemblyai",
-  "version": "4.36.3",
+  "version": "4.36.4",
   "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.",
   "engines": {
     "node": ">=18"

src/services/streaming/service.ts

@@ -20,6 +20,7 @@ import {
   StreamingForceEndpoint,
   StreamingKeepAlive,
   WarningEvent,
+  HeartbeatEvent,
 } from "../..";
 import type { VadDetector, VadFrame } from "../../types/streaming/dual-channel";
 import { EnergyVad } from "./energy-vad";
@@ -164,10 +165,16 @@ export class StreamingTranscriber {
       throw new Error("API key or temporary token is required.");
     }
 
-    const isOpus = params.encoding === "opus" || params.encoding === "ogg_opus";
-    if (params.sampleRate === undefined && (!isOpus || params.channels)) {
+    const isSelfDescribing =
+      params.encoding === "opus" ||
+      params.encoding === "ogg_opus" ||
+      params.encoding === "aac";
+    if (
+      params.sampleRate === undefined &&
+      (!isSelfDescribing || params.channels)
+    ) {
       throw new Error(
-        '`sampleRate` is required; it may only be omitted when `encoding` is "opus" or "ogg_opus" (the Opus stream is self-describing) and dual-channel mode is not used.',
+        '`sampleRate` is required; it may only be omitted when `encoding` is "opus", "ogg_opus", or "aac" (these streams are self-describing) and dual-channel mode is not used.',
       );
     }
 
@@ -283,6 +290,13 @@ export class StreamingTranscriber {
       searchParams.set("format_turns", this.params.formatTurns.toString());
     }
 
+    if (this.params.sessionHeartbeat !== undefined) {
+      searchParams.set(
+        "session_heartbeat",
+        this.params.sessionHeartbeat.toString(),
+      );
+    }
+
     if (this.params.encoding) {
       searchParams.set("encoding", this.params.encoding.toString());
     }
@@ -474,6 +488,7 @@ export class StreamingTranscriber {
     listener: (event: SpeakerRevisionEvent) => void,
   ): void;
   on(event: "warning", listener: (event: WarningEvent) => void): void;
+  on(event: "heartbeat", listener: (event: HeartbeatEvent) => void): void;
   on(event: "vad", listener: (event: VadFrame) => void): void;
   on(event: "error", listener: (error: Error) => void): void;
   on(event: "close", listener: (code: number, reason: string) => void): void;
@@ -693,6 +708,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c
             this.listeners.warning?.(warning);
             break;
           }
+          case "Heartbeat": {
+            this.listeners.heartbeat?.(message);
+            break;
+          }
           case "Termination": {
             this.sessionTerminatedResolve?.();
             break;

src/types/asyncapi.generated.ts

@@ -25,7 +25,12 @@ export type AudioData = ArrayBufferLike;
  * The encoding of the audio data
  * @defaultValue "pcm_s16"le
  */
-export type AudioEncoding = "pcm_s16le" | "pcm_mulaw" | "opus" | "ogg_opus";
+export type AudioEncoding =
+  | "pcm_s16le"
+  | "pcm_mulaw"
+  | "opus"
+  | "ogg_opus"
+  | "aac";
 
 /**
  * Configure the threshold for how long to wait before ending an utterance. Default is 700ms.

src/types/streaming/index.ts

@@ -94,8 +94,8 @@ export type StreamingTranscriberParams = {
   connectionRetryDelay?: number;
   /**
    * Required for PCM encodings (and for dual-channel mode). May be omitted
-   * for Opus encodings (`opus`, `ogg_opus`) — the stream is self-describing
-   * and the server ignores the value.
+   * for self-describing encodings (`opus`, `ogg_opus`, `aac`) — the stream
+   * carries its own rate and the server ignores the value.
    */
   sampleRate?: number;
   encoding?: AudioEncoding;
@@ -108,6 +108,7 @@ export type StreamingTranscriberParams = {
   maxTurnSilence?: number;
   vadThreshold?: number;
   formatTurns?: boolean;
+  sessionHeartbeat?: boolean;
   filterProfanity?: boolean;
   keyterms?: string[];
   keytermsPrompt?: string[];
@@ -184,6 +185,7 @@ export type StreamingEvents =
   | "llmGatewayResponse"
   | "speakerRevision"
   | "warning"
+  | "heartbeat"
   | "vad"
   | "error";
 
@@ -195,6 +197,7 @@ export type StreamingListeners = {
   llmGatewayResponse?: (event: LLMGatewayResponseEvent) => void;
   speakerRevision?: (event: SpeakerRevisionEvent) => void;
   warning?: (event: WarningEvent) => void;
+  heartbeat?: (event: HeartbeatEvent) => void;
   vad?: (event: VadFrame) => void;
   error?: (error: Error) => void;
 };
@@ -370,6 +373,7 @@ export type StreamingUpdateConfiguration = {
   max_turn_silence?: number;
   vad_threshold?: number;
   format_turns?: boolean;
+  session_heartbeat?: boolean;
   keyterms_prompt?: string[];
   prompt?: string;
   agent_context?: string;
@@ -404,6 +408,14 @@ export type WarningEvent = {
   warning: string;
 };
 
+export type HeartbeatEvent = {
+  type: "Heartbeat";
+  total_audio_received_ms: number;
+  total_duration_ms: number;
+  realtime_factor: number;
+  max_speech_probability: number;
+};
+
 export type LLMGatewayResponseEvent = {
   type: "LLMGatewayResponse";
   turn_order: number;
@@ -443,7 +455,8 @@ export type StreamingEventMessage =
   | LLMGatewayResponseEvent
   | SpeakerRevisionEvent
   | ErrorEvent
-  | WarningEvent;
+  | WarningEvent
+  | HeartbeatEvent;
 
 export type StreamingOperationMessage =
   | StreamingUpdateConfiguration

tests/unit/streaming.test.ts

@@ -38,6 +38,18 @@ async function close(rt: StreamingTranscriber, server: WS) {
   await server.closed;
 }
 
+// Read the actual URL the transcriber connected with. `jest-websocket-mock`
+// (via mock-socket) trims the query string before matching a server, so
+// `new WS(fullUrl)` cannot verify query params — the connection matches on the
+// base URL regardless. Inspect the real URL the client passed to the socket
+// instead, which is `connectionUrl().toString()`.
+function connectUrlParams(transcriber: StreamingTranscriber): URLSearchParams {
+  const socketUrl = (transcriber as unknown as { socket?: { url: string } })
+    .socket?.url;
+  if (!socketUrl) throw new Error("transcriber has no open socket");
+  return new URL(socketUrl).searchParams;
+}
+
 describe("streaming", () => {
   beforeEach(async () => {
     server = new WS(websocketBaseUrl);
@@ -408,7 +420,7 @@ describe("streaming", () => {
     },
   );
 
-  it.each(["opus", "ogg_opus"] as const)(
+  it.each(["opus", "ogg_opus", "aac"] as const)(
     "should allow omitting sampleRate for %s encoding",
     async (encoding) => {
       await cleanup();
@@ -726,4 +738,124 @@ describe("streaming", () => {
     expect(onOpen).toHaveBeenCalled();
     // Leaves rt/server connected for the shared afterEach cleanup.
   });
+
+  it("should include encoding=aac in connection URL and allow omitting sampleRate", async () => {
+    await cleanup();
+    WS.clean();
+
+    // AAC (ADTS) is self-describing, so sampleRate is omitted here — this also
+    // guards that the constructor's sampleRate validation exempts aac.
+    const wsUrl = `${websocketBaseUrl}?token=123&encoding=aac`;
+    server = new WS(wsUrl);
+    rt = new StreamingTranscriber({
+      websocketBaseUrl,
+      token: "123",
+      encoding: "aac",
+    });
+    onOpen = jest.fn();
+    rt.on("open", onOpen);
+    await connect(rt, server);
+
+    const params = connectUrlParams(rt);
+    expect(params.get("encoding")).toBe("aac");
+    expect(params.has("sample_rate")).toBe(false);
+  });
+
+  it("should include session_heartbeat=true in connection URL", async () => {
+    await cleanup();
+    WS.clean();
+
+    const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&session_heartbeat=true`;
+    server = new WS(wsUrl);
+    rt = new StreamingTranscriber({
+      websocketBaseUrl,
+      token: "123",
+      sampleRate: 16_000,
+      sessionHeartbeat: true,
+    });
+    onOpen = jest.fn();
+    rt.on("open", onOpen);
+    await connect(rt, server);
+
+    expect(connectUrlParams(rt).get("session_heartbeat")).toBe("true");
+  });
+
+  it("should include session_heartbeat=false in connection URL when explicitly disabled", async () => {
+    await cleanup();
+    WS.clean();
+
+    // Serialization uses `!== undefined`, so an explicit `false` is still emitted.
+    const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&session_heartbeat=false`;
+    server = new WS(wsUrl);
+    rt = new StreamingTranscriber({
+      websocketBaseUrl,
+      token: "123",
+      sampleRate: 16_000,
+      sessionHeartbeat: false,
+    });
+    onOpen = jest.fn();
+    rt.on("open", onOpen);
+    await connect(rt, server);
+
+    expect(connectUrlParams(rt).get("session_heartbeat")).toBe("false");
+  });
+
+  it("should omit session_heartbeat from connection URL when unset", async () => {
+    await cleanup();
+    WS.clean();
+
+    const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000`;
+    server = new WS(wsUrl);
+    rt = new StreamingTranscriber({
+      websocketBaseUrl,
+      token: "123",
+      sampleRate: 16_000,
+    });
+    onOpen = jest.fn();
+    rt.on("open", onOpen);
+    await connect(rt, server);
+
+    expect(connectUrlParams(rt).has("session_heartbeat")).toBe(false);
+  });
+
+  it("should parse and dispatch Heartbeat event", async () => {
+    const handler = jest.fn();
+    const heartbeatPromise = new Promise<{
+      type: string;
+      total_audio_received_ms: number;
+      total_duration_ms: number;
+      realtime_factor: number;
+      max_speech_probability: number;
+    }>((resolve) => {
+      rt.on("heartbeat", (event) => {
+        handler(event);
+        resolve(event);
+      });
+    });
+
+    const heartbeat = {
+      type: "Heartbeat",
+      total_audio_received_ms: 45000,
+      total_duration_ms: 45205,
+      realtime_factor: 0.9964,
+      max_speech_probability: 0.999954,
+    };
+
+    server.send(JSON.stringify(heartbeat));
+
+    const event = await heartbeatPromise;
+    expect(handler).toHaveBeenCalledTimes(1);
+    // snake_case wire payload is passed through verbatim.
+    expect(event).toEqual(heartbeat);
+  });
+
+  it("should include session_heartbeat in updateConfiguration message", async () => {
+    rt.updateConfiguration({ session_heartbeat: true });
+    await expect(server).toReceiveMessage(
+      JSON.stringify({
+        type: "UpdateConfiguration",
+        session_heartbeat: true,
+      }),
+    );
+  });
 });