From 8f85e2d9fd0eda09f5af0057f6d4362a01b72bc1 Mon Sep 17 00:00:00 2001 From: AssemblyAI Date: Wed, 22 Jul 2026 11:32:40 -0600 Subject: [PATCH] Project import generated by Copybara. GitOrigin-RevId: f3e048b62ae729f0575190e81e3f0ac4faf44eea --- CHANGELOG.md | 5 ++ package.json | 2 +- src/services/streaming/service.ts | 25 +++++- src/types/asyncapi.generated.ts | 7 +- src/types/streaming/index.ts | 19 ++++- tests/unit/streaming.test.ts | 134 +++++++++++++++++++++++++++++- 6 files changed, 183 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fbb49..17af2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [4.36.4] + +- Add `aac` to the streaming `encoding` options — accepts an AAC stream in ADTS framing. Like `opus`/`ogg_opus`, AAC is self-describing, so `sampleRate` is optional for it (it remains required for PCM encodings and for dual-channel mode) +- Add opt-in `sessionHeartbeat` streaming param — when enabled, the server emits a periodic `Heartbeat` message, surfaced via the new `heartbeat` event with `total_audio_received_ms`, `total_duration_ms`, `realtime_factor`, and `max_speech_probability`. Requires the `universal-3-5-pro` streaming model + ## [4.36.3] - Target the canonical `/v1` sync API routes (`/v1/transcribe`, `/v1/warm`); the unprefixed paths remain served for older SDK versions diff --git a/package.json b/package.json index 20a6c43..5d3de18 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "assemblyai", - "version": "4.36.3", + "version": "4.36.4", "description": "The AssemblyAI JavaScript SDK provides an easy-to-use interface for interacting with the AssemblyAI API, which supports async and real-time transcription, as well as the latest LeMUR models.", "engines": { "node": ">=18" diff --git a/src/services/streaming/service.ts b/src/services/streaming/service.ts index f894da4..610582a 100644 --- a/src/services/streaming/service.ts +++ b/src/services/streaming/service.ts @@ -20,6 +20,7 @@ import { StreamingForceEndpoint, StreamingKeepAlive, WarningEvent, + HeartbeatEvent, } from "../.."; import type { VadDetector, VadFrame } from "../../types/streaming/dual-channel"; import { EnergyVad } from "./energy-vad"; @@ -164,10 +165,16 @@ export class StreamingTranscriber { throw new Error("API key or temporary token is required."); } - const isOpus = params.encoding === "opus" || params.encoding === "ogg_opus"; - if (params.sampleRate === undefined && (!isOpus || params.channels)) { + const isSelfDescribing = + params.encoding === "opus" || + params.encoding === "ogg_opus" || + params.encoding === "aac"; + if ( + params.sampleRate === undefined && + (!isSelfDescribing || params.channels) + ) { throw new Error( - '`sampleRate` is required; it may only be omitted when `encoding` is "opus" or "ogg_opus" (the Opus stream is self-describing) and dual-channel mode is not used.', + '`sampleRate` is required; it may only be omitted when `encoding` is "opus", "ogg_opus", or "aac" (these streams are self-describing) and dual-channel mode is not used.', ); } @@ -283,6 +290,13 @@ export class StreamingTranscriber { searchParams.set("format_turns", this.params.formatTurns.toString()); } + if (this.params.sessionHeartbeat !== undefined) { + searchParams.set( + "session_heartbeat", + this.params.sessionHeartbeat.toString(), + ); + } + if (this.params.encoding) { searchParams.set("encoding", this.params.encoding.toString()); } @@ -474,6 +488,7 @@ export class StreamingTranscriber { listener: (event: SpeakerRevisionEvent) => void, ): void; on(event: "warning", listener: (event: WarningEvent) => void): void; + on(event: "heartbeat", listener: (event: HeartbeatEvent) => void): void; on(event: "vad", listener: (event: VadFrame) => void): void; on(event: "error", listener: (error: Error) => void): void; on(event: "close", listener: (code: number, reason: string) => void): void; @@ -693,6 +708,10 @@ Learn more at https://github.com/AssemblyAI/assemblyai-node-sdk/blob/main/docs/c this.listeners.warning?.(warning); break; } + case "Heartbeat": { + this.listeners.heartbeat?.(message); + break; + } case "Termination": { this.sessionTerminatedResolve?.(); break; diff --git a/src/types/asyncapi.generated.ts b/src/types/asyncapi.generated.ts index 5b70c8a..29dafc3 100644 --- a/src/types/asyncapi.generated.ts +++ b/src/types/asyncapi.generated.ts @@ -25,7 +25,12 @@ export type AudioData = ArrayBufferLike; * The encoding of the audio data * @defaultValue "pcm_s16"le */ -export type AudioEncoding = "pcm_s16le" | "pcm_mulaw" | "opus" | "ogg_opus"; +export type AudioEncoding = + | "pcm_s16le" + | "pcm_mulaw" + | "opus" + | "ogg_opus" + | "aac"; /** * Configure the threshold for how long to wait before ending an utterance. Default is 700ms. diff --git a/src/types/streaming/index.ts b/src/types/streaming/index.ts index 20f4d3c..df22b49 100644 --- a/src/types/streaming/index.ts +++ b/src/types/streaming/index.ts @@ -94,8 +94,8 @@ export type StreamingTranscriberParams = { connectionRetryDelay?: number; /** * Required for PCM encodings (and for dual-channel mode). May be omitted - * for Opus encodings (`opus`, `ogg_opus`) — the stream is self-describing - * and the server ignores the value. + * for self-describing encodings (`opus`, `ogg_opus`, `aac`) — the stream + * carries its own rate and the server ignores the value. */ sampleRate?: number; encoding?: AudioEncoding; @@ -108,6 +108,7 @@ export type StreamingTranscriberParams = { maxTurnSilence?: number; vadThreshold?: number; formatTurns?: boolean; + sessionHeartbeat?: boolean; filterProfanity?: boolean; keyterms?: string[]; keytermsPrompt?: string[]; @@ -184,6 +185,7 @@ export type StreamingEvents = | "llmGatewayResponse" | "speakerRevision" | "warning" + | "heartbeat" | "vad" | "error"; @@ -195,6 +197,7 @@ export type StreamingListeners = { llmGatewayResponse?: (event: LLMGatewayResponseEvent) => void; speakerRevision?: (event: SpeakerRevisionEvent) => void; warning?: (event: WarningEvent) => void; + heartbeat?: (event: HeartbeatEvent) => void; vad?: (event: VadFrame) => void; error?: (error: Error) => void; }; @@ -370,6 +373,7 @@ export type StreamingUpdateConfiguration = { max_turn_silence?: number; vad_threshold?: number; format_turns?: boolean; + session_heartbeat?: boolean; keyterms_prompt?: string[]; prompt?: string; agent_context?: string; @@ -404,6 +408,14 @@ export type WarningEvent = { warning: string; }; +export type HeartbeatEvent = { + type: "Heartbeat"; + total_audio_received_ms: number; + total_duration_ms: number; + realtime_factor: number; + max_speech_probability: number; +}; + export type LLMGatewayResponseEvent = { type: "LLMGatewayResponse"; turn_order: number; @@ -443,7 +455,8 @@ export type StreamingEventMessage = | LLMGatewayResponseEvent | SpeakerRevisionEvent | ErrorEvent - | WarningEvent; + | WarningEvent + | HeartbeatEvent; export type StreamingOperationMessage = | StreamingUpdateConfiguration diff --git a/tests/unit/streaming.test.ts b/tests/unit/streaming.test.ts index 9b1f694..8cf0b90 100644 --- a/tests/unit/streaming.test.ts +++ b/tests/unit/streaming.test.ts @@ -38,6 +38,18 @@ async function close(rt: StreamingTranscriber, server: WS) { await server.closed; } +// Read the actual URL the transcriber connected with. `jest-websocket-mock` +// (via mock-socket) trims the query string before matching a server, so +// `new WS(fullUrl)` cannot verify query params — the connection matches on the +// base URL regardless. Inspect the real URL the client passed to the socket +// instead, which is `connectionUrl().toString()`. +function connectUrlParams(transcriber: StreamingTranscriber): URLSearchParams { + const socketUrl = (transcriber as unknown as { socket?: { url: string } }) + .socket?.url; + if (!socketUrl) throw new Error("transcriber has no open socket"); + return new URL(socketUrl).searchParams; +} + describe("streaming", () => { beforeEach(async () => { server = new WS(websocketBaseUrl); @@ -408,7 +420,7 @@ describe("streaming", () => { }, ); - it.each(["opus", "ogg_opus"] as const)( + it.each(["opus", "ogg_opus", "aac"] as const)( "should allow omitting sampleRate for %s encoding", async (encoding) => { await cleanup(); @@ -726,4 +738,124 @@ describe("streaming", () => { expect(onOpen).toHaveBeenCalled(); // Leaves rt/server connected for the shared afterEach cleanup. }); + + it("should include encoding=aac in connection URL and allow omitting sampleRate", async () => { + await cleanup(); + WS.clean(); + + // AAC (ADTS) is self-describing, so sampleRate is omitted here — this also + // guards that the constructor's sampleRate validation exempts aac. + const wsUrl = `${websocketBaseUrl}?token=123&encoding=aac`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + encoding: "aac", + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + const params = connectUrlParams(rt); + expect(params.get("encoding")).toBe("aac"); + expect(params.has("sample_rate")).toBe(false); + }); + + it("should include session_heartbeat=true in connection URL", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&session_heartbeat=true`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + sessionHeartbeat: true, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + expect(connectUrlParams(rt).get("session_heartbeat")).toBe("true"); + }); + + it("should include session_heartbeat=false in connection URL when explicitly disabled", async () => { + await cleanup(); + WS.clean(); + + // Serialization uses `!== undefined`, so an explicit `false` is still emitted. + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000&session_heartbeat=false`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + sessionHeartbeat: false, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + expect(connectUrlParams(rt).get("session_heartbeat")).toBe("false"); + }); + + it("should omit session_heartbeat from connection URL when unset", async () => { + await cleanup(); + WS.clean(); + + const wsUrl = `${websocketBaseUrl}?token=123&sample_rate=16000`; + server = new WS(wsUrl); + rt = new StreamingTranscriber({ + websocketBaseUrl, + token: "123", + sampleRate: 16_000, + }); + onOpen = jest.fn(); + rt.on("open", onOpen); + await connect(rt, server); + + expect(connectUrlParams(rt).has("session_heartbeat")).toBe(false); + }); + + it("should parse and dispatch Heartbeat event", async () => { + const handler = jest.fn(); + const heartbeatPromise = new Promise<{ + type: string; + total_audio_received_ms: number; + total_duration_ms: number; + realtime_factor: number; + max_speech_probability: number; + }>((resolve) => { + rt.on("heartbeat", (event) => { + handler(event); + resolve(event); + }); + }); + + const heartbeat = { + type: "Heartbeat", + total_audio_received_ms: 45000, + total_duration_ms: 45205, + realtime_factor: 0.9964, + max_speech_probability: 0.999954, + }; + + server.send(JSON.stringify(heartbeat)); + + const event = await heartbeatPromise; + expect(handler).toHaveBeenCalledTimes(1); + // snake_case wire payload is passed through verbatim. + expect(event).toEqual(heartbeat); + }); + + it("should include session_heartbeat in updateConfiguration message", async () => { + rt.updateConfiguration({ session_heartbeat: true }); + await expect(server).toReceiveMessage( + JSON.stringify({ + type: "UpdateConfiguration", + session_heartbeat: true, + }), + ); + }); });