diff --git a/COVERAGE.md b/COVERAGE.md index 949d865..b88e8ec 100644 --- a/COVERAGE.md +++ b/COVERAGE.md @@ -20,8 +20,9 @@ suffix). See `agents/monorepo.md` for the convention. ## Voice Agents -Voice Agents has no SDK yet — all recipes are native REST (`-rest`). Bare names are -reserved for SDK versions. +Voice Agents recipes are native REST (`-rest`); `realtime-conversation` previews the +SDK's upcoming realtime support in both TypeScript and Python (pinned to the SDK's +pre-release branch until it ships). | Recipe | TypeScript | Python | Rust | | ---------------------------------------------- | :--------: | :----: | :--: | @@ -29,7 +30,7 @@ reserved for SDK versions. | manage agents (CRUD) | ✅ | ✅ | ⬜ | | inspect conversation transcript | ✅ | ✅ | ⬜ | | browser widget embed | ⬜ | — | — | -| live audio session (LiveKit room) | ⬜ | ⬜ | ⬜ | +| realtime conversation (stream audio in/out) | ✅ | ✅ | ⬜ | > Rust columns are tracked for planning but Rust tooling is **not enabled yet** — see > `agents/monorepo.md`. diff --git a/README.md b/README.md index bab0d62..4bd7158 100644 --- a/README.md +++ b/README.md @@ -43,17 +43,21 @@ pnpm install && pnpm start # writes output.mp3 ### Voice Agents -Voice Agents has no SDK yet, so every recipe is **native REST** (`-rest` suffix). The bare -names are reserved for SDK versions once an SDK adds Voice Agents support. - -| Recipe | Language | Description | -| ---------------------------------------------------------------------------------------------- | ---------- | ----------------------------------------------------- | -| [quickstart-rest](./recipes/voice-agents/typescript/quickstart-rest) | TypeScript | Create an agent and open a live conversation session. | -| [quickstart-rest](./recipes/voice-agents/python/quickstart-rest) | Python | Create an agent and open a live conversation session. | -| [manage-agents-rest](./recipes/voice-agents/typescript/manage-agents-rest) | TypeScript | Agent CRUD: create, list, get, update, delete. | -| [manage-agents-rest](./recipes/voice-agents/python/manage-agents-rest) | Python | Agent CRUD: create, list, get, update, delete. | -| [conversation-transcript-rest](./recipes/voice-agents/typescript/conversation-transcript-rest) | TypeScript | Start a conversation and read its transcript. | -| [conversation-transcript-rest](./recipes/voice-agents/python/conversation-transcript-rest) | Python | Start a conversation and read its transcript. | +Voice Agents recipes are **native REST** (`-rest` suffix) today. The first **SDK** recipe, +`realtime-conversation`, previews the SDK's upcoming realtime support in both TypeScript and +Python — it is pinned to the SDK's pre-release branch until that ships, then switches to the +published version. + +| Recipe | Language | Description | +| ---------------------------------------------------------------------------------------------- | ---------- | ------------------------------------------------------------------------------------ | +| [quickstart-rest](./recipes/voice-agents/typescript/quickstart-rest) | TypeScript | Create an agent and open a live conversation session. | +| [quickstart-rest](./recipes/voice-agents/python/quickstart-rest) | Python | Create an agent and open a live conversation session. | +| [manage-agents-rest](./recipes/voice-agents/typescript/manage-agents-rest) | TypeScript | Agent CRUD: create, list, get, update, delete. | +| [manage-agents-rest](./recipes/voice-agents/python/manage-agents-rest) | Python | Agent CRUD: create, list, get, update, delete. | +| [conversation-transcript-rest](./recipes/voice-agents/typescript/conversation-transcript-rest) | TypeScript | Start a conversation and read its transcript. | +| [conversation-transcript-rest](./recipes/voice-agents/python/conversation-transcript-rest) | Python | Start a conversation and read its transcript. | +| [realtime-conversation](./recipes/voice-agents/typescript/realtime-conversation) | TypeScript | Stream audio to a voice agent in real time and save its spoken reply (SDK realtime). | +| [realtime-conversation](./recipes/voice-agents/python/realtime-conversation) | Python | Stream audio to a voice agent in real time and save its spoken reply (SDK realtime). | See [`COVERAGE.md`](./COVERAGE.md) for the full product × language matrix and what's planned. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6524fde..4f008a0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -189,8 +189,33 @@ importers: specifier: 'catalog:' version: 5.9.3 + recipes/voice-agents/typescript/realtime-conversation: + dependencies: + '@speechify/api': + specifier: github:SpeechifyInc/speechify-api-sdk-typescript#realtime-connect + version: https://codeload.github.com/SpeechifyInc/speechify-api-sdk-typescript/tar.gz/292183c866cfdede54b86a9646b9ddf8797f92a5 + dotenv: + specifier: 'catalog:' + version: 16.6.1 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 22.19.21 + tsx: + specifier: 'catalog:' + version: 4.22.4 + typescript: + specifier: 'catalog:' + version: 5.9.3 + packages: + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} + + '@datastructures-js/deque@1.0.8': + resolution: {integrity: sha512-PSBhJ2/SmeRPRHuBv7i/fHWIdSC3JTyq56qb+Rq0wjOagi0/fdV5/B/3Md5zFZus/W6OkSPMaxMKKMNMrSmubg==} + '@esbuild/aix-ppc64@0.28.1': resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} engines: {node: '>=18'} @@ -347,9 +372,60 @@ packages: cpu: [x64] os: [win32] + '@livekit/mutex@1.1.1': + resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} + + '@livekit/rtc-ffi-bindings-darwin-arm64@0.12.60': + resolution: {integrity: sha512-YHXqybkYfaTc3txJXXWoVogiSP3yKJdkaZlIlZ6IDMGnN9elUoHDYU+ZSn/rbdGu0pp4HUOzffXkbkItN735Bw==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [darwin] + + '@livekit/rtc-ffi-bindings-darwin-x64@0.12.60': + resolution: {integrity: sha512-SkPPWE2/nb2BAXrCWP6+vaR2I4EeyG3Vv+csUaa1EvDVMbFqBHWqNVTQcx/ChgecbYB9dIFZHVYpfjbFkVd84g==} + engines: {node: '>= 18'} + cpu: [x64] + os: [darwin] + + '@livekit/rtc-ffi-bindings-linux-arm64-gnu@0.12.60': + resolution: {integrity: sha512-8umeMn9p/VZ41EGty1qX9zPV5mfGxCioYKeUnALpf8AKqT/yXDjnog1VkS5f8gFX/zY7HDaeE+s60nZXaOZJbw==} + engines: {node: '>= 18'} + cpu: [arm64] + os: [linux] + + '@livekit/rtc-ffi-bindings-linux-x64-gnu@0.12.60': + resolution: {integrity: sha512-ttWrR/e8Ghaa9I+LaStxK8lh+aA9QBz6Dge6eXyKwTrAMHwHEtL2Rnf1rHQTiwadeH7AoytpfWf5FZl/OelLaQ==} + engines: {node: '>= 18'} + cpu: [x64] + os: [linux] + + '@livekit/rtc-ffi-bindings-win32-x64-msvc@0.12.60': + resolution: {integrity: sha512-HfOBEf3rmpsG7hU3/BM9x2jnkVwKFve2v3cyjxlk41d6OkCthYb+g/ULEPFBKPafYyepDwcd2c8MDo5fMM+4Zw==} + engines: {node: '>= 18'} + cpu: [x64] + os: [win32] + + '@livekit/rtc-ffi-bindings@0.12.60': + resolution: {integrity: sha512-ZJD2DNoHfR8PzKeyDMH6i1zKpk7S4LlrQDIZvisxj6HPaJnKofzSssNMF8fpGFvVCZ844kbcOFogRPgHFno82w==} + engines: {node: '>= 18'} + + '@livekit/rtc-node@0.13.29': + resolution: {integrity: sha512-3/mhTVW3zEa8u0l2UzLe74CyDxaz/1Fqrss+monBJARYHyMGMiDlnKAwDT2LiFkRy0xBjl2QY8Yldr0icfPUkw==} + engines: {node: '>= 18'} + + '@livekit/typed-emitter@3.0.0': + resolution: {integrity: sha512-9bl0k4MgBPZu3Qu3R3xy12rmbW17e3bE9yf4YY85gJIQ3ezLEj/uzpKHWBsLaDoL5Mozz8QCgggwIBudYQWeQg==} + + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@speechify/api@1.0.1': resolution: {integrity: sha512-B6KT965FjoR8R/2UeS7TEnmnxfleVqR7ORIvrIDdv5kDe6H4bL5jSA2RcZbg5SfOAgJRp+eV1lqT/M11LBKhvw==} + '@speechify/api@https://codeload.github.com/SpeechifyInc/speechify-api-sdk-typescript/tar.gz/292183c866cfdede54b86a9646b9ddf8797f92a5': + resolution: {tarball: https://codeload.github.com/SpeechifyInc/speechify-api-sdk-typescript/tar.gz/292183c866cfdede54b86a9646b9ddf8797f92a5} + version: 1.0.2 + '@types/node@22.19.21': resolution: {integrity: sha512-VMeFBSCKQKmm2swI2kW51SFusDqekC6q9trBCvJ/JliDchFSuoYYKN7yVNjPthP1HKZcx3U1gI/wTcEBjEFKTA==} @@ -360,6 +436,10 @@ packages: asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} @@ -374,10 +454,16 @@ packages: resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} engines: {node: '>= 0.4'} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -390,6 +476,9 @@ packages: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + es-define-property@1.0.1: resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} engines: {node: '>= 0.4'} @@ -419,6 +508,12 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + fast-copy@4.0.3: + resolution: {integrity: sha512-58apWr0GUiDFM8+3afrO6eYwJBn9ZAhDOzG3L+/9llab/haCARS2UIfffmOurYLwbgDRs8n0rfr6qAAPEAuAQw==} + + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + form-data-encoder@4.1.0: resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==} engines: {node: '>= 18'} @@ -463,9 +558,16 @@ packages: resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} engines: {node: '>= 0.4'} + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + js-base64@3.7.7: resolution: {integrity: sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw==} @@ -481,6 +583,9 @@ packages: resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} engines: {node: '>= 0.6'} + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -494,26 +599,70 @@ packages: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + pino-abstract-transport@2.0.0: + resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} + + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-pretty@13.1.3: + resolution: {integrity: sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==} + hasBin: true + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@9.14.0: + resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} + hasBin: true + prettier@3.8.4: resolution: {integrity: sha512-N2MylSdi48+5N/6S5j+maeHbUSIzzZ5uOcX5Hm4QpV8Dkb1HFjfAKTKX6yNPJQD9AhcT3ifHNB66tWTTJDi11Q==} engines: {node: '>=14'} hasBin: true + process-warning@5.0.0: + resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + process@0.11.10: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + qs@6.15.2: resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==} engines: {node: '>=0.6'} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + readable-stream@4.7.0: resolution: {integrity: sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + side-channel-list@1.0.1: resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==} engines: {node: '>= 0.4'} @@ -530,9 +679,23 @@ packages: resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} engines: {node: '>= 0.4'} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + string_decoder@1.3.0: resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + strip-json-comments@5.0.3: + resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} + engines: {node: '>=14.16'} + + thread-stream@3.2.0: + resolution: {integrity: sha512-zLBvqpwr4Esa0kRjcrzGU6zL25lePWaCLMx0RQFrmteozIfeNdaMLpG5U7PeHzvlFkAWaRKA9/KVW4F60iB+qw==} + tr46@0.0.3: resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} @@ -558,8 +721,15 @@ packages: whatwg-url@5.0.0: resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + snapshots: + '@bufbuild/protobuf@1.10.1': {} + + '@datastructures-js/deque@1.0.8': {} + '@esbuild/aix-ppc64@0.28.1': optional: true @@ -638,6 +808,46 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true + '@livekit/mutex@1.1.1': {} + + '@livekit/rtc-ffi-bindings-darwin-arm64@0.12.60': + optional: true + + '@livekit/rtc-ffi-bindings-darwin-x64@0.12.60': + optional: true + + '@livekit/rtc-ffi-bindings-linux-arm64-gnu@0.12.60': + optional: true + + '@livekit/rtc-ffi-bindings-linux-x64-gnu@0.12.60': + optional: true + + '@livekit/rtc-ffi-bindings-win32-x64-msvc@0.12.60': + optional: true + + '@livekit/rtc-ffi-bindings@0.12.60': + dependencies: + '@bufbuild/protobuf': 1.10.1 + optionalDependencies: + '@livekit/rtc-ffi-bindings-darwin-arm64': 0.12.60 + '@livekit/rtc-ffi-bindings-darwin-x64': 0.12.60 + '@livekit/rtc-ffi-bindings-linux-arm64-gnu': 0.12.60 + '@livekit/rtc-ffi-bindings-linux-x64-gnu': 0.12.60 + '@livekit/rtc-ffi-bindings-win32-x64-msvc': 0.12.60 + + '@livekit/rtc-node@0.13.29': + dependencies: + '@datastructures-js/deque': 1.0.8 + '@livekit/mutex': 1.1.1 + '@livekit/rtc-ffi-bindings': 0.12.60 + '@livekit/typed-emitter': 3.0.0 + pino: 9.14.0 + pino-pretty: 13.1.3 + + '@livekit/typed-emitter@3.0.0': {} + + '@pinojs/redact@0.4.0': {} + '@speechify/api@1.0.1': dependencies: form-data: 4.0.6 @@ -651,6 +861,20 @@ snapshots: transitivePeerDependencies: - encoding + '@speechify/api@https://codeload.github.com/SpeechifyInc/speechify-api-sdk-typescript/tar.gz/292183c866cfdede54b86a9646b9ddf8797f92a5': + dependencies: + '@livekit/rtc-node': 0.13.29 + form-data: 4.0.6 + form-data-encoder: 4.1.0 + formdata-node: 6.0.3 + js-base64: 3.7.7 + node-fetch: 2.7.0 + qs: 6.15.2 + readable-stream: 4.7.0 + url-join: 4.0.1 + transitivePeerDependencies: + - encoding + '@types/node@22.19.21': dependencies: undici-types: 6.21.0 @@ -661,6 +885,8 @@ snapshots: asynckit@0.4.0: {} + atomic-sleep@1.0.0: {} + base64-js@1.5.1: {} buffer@6.0.3: @@ -678,10 +904,14 @@ snapshots: call-bind-apply-helpers: 1.0.2 get-intrinsic: 1.3.0 + colorette@2.0.20: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 + dateformat@4.6.3: {} + delayed-stream@1.0.0: {} dotenv@16.6.1: {} @@ -692,6 +922,10 @@ snapshots: es-errors: 1.3.0 gopd: 1.2.0 + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + es-define-property@1.0.1: {} es-errors@1.3.0: {} @@ -740,6 +974,10 @@ snapshots: events@3.3.0: {} + fast-copy@4.0.3: {} + + fast-safe-stringify@2.1.1: {} + form-data-encoder@4.1.0: {} form-data@4.0.6: @@ -787,8 +1025,12 @@ snapshots: dependencies: function-bind: 1.1.2 + help-me@5.0.0: {} + ieee754@1.2.1: {} + joycon@3.1.1: {} + js-base64@3.7.7: {} math-intrinsics@1.1.0: {} @@ -799,20 +1041,77 @@ snapshots: dependencies: mime-db: 1.52.0 + minimist@1.2.8: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 object-inspect@1.13.4: {} + on-exit-leak-free@2.1.2: {} + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + pino-abstract-transport@2.0.0: + dependencies: + split2: 4.2.0 + + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-pretty@13.1.3: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 4.0.3 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pump: 3.0.4 + secure-json-parse: 4.1.0 + sonic-boom: 4.2.1 + strip-json-comments: 5.0.3 + + pino-std-serializers@7.1.0: {} + + pino@9.14.0: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 2.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 3.2.0 + prettier@3.8.4: {} + process-warning@5.0.0: {} + process@0.11.10: {} + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + qs@6.15.2: dependencies: side-channel: 1.1.1 + quick-format-unescaped@4.0.4: {} + readable-stream@4.7.0: dependencies: abort-controller: 3.0.0 @@ -821,8 +1120,14 @@ snapshots: process: 0.11.10 string_decoder: 1.3.0 + real-require@0.2.0: {} + safe-buffer@5.2.1: {} + safe-stable-stringify@2.5.0: {} + + secure-json-parse@4.1.0: {} + side-channel-list@1.0.1: dependencies: es-errors: 1.3.0 @@ -851,10 +1156,22 @@ snapshots: side-channel-map: 1.0.1 side-channel-weakmap: 1.0.2 + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + + split2@4.2.0: {} + string_decoder@1.3.0: dependencies: safe-buffer: 5.2.1 + strip-json-comments@5.0.3: {} + + thread-stream@3.2.0: + dependencies: + real-require: 0.2.0 + tr46@0.0.3: {} tsx@4.22.4: @@ -875,3 +1192,5 @@ snapshots: dependencies: tr46: 0.0.3 webidl-conversions: 3.0.1 + + wrappy@1.0.2: {} diff --git a/recipes/voice-agents/python/realtime-conversation/.env.example b/recipes/voice-agents/python/realtime-conversation/.env.example new file mode 100644 index 0000000..4cb51e4 --- /dev/null +++ b/recipes/voice-agents/python/realtime-conversation/.env.example @@ -0,0 +1 @@ +SPEECHIFY_API_KEY= diff --git a/recipes/voice-agents/python/realtime-conversation/README.md b/recipes/voice-agents/python/realtime-conversation/README.md new file mode 100644 index 0000000..9b4bf5e --- /dev/null +++ b/recipes/voice-agents/python/realtime-conversation/README.md @@ -0,0 +1,33 @@ +# Voice Agents: real-time conversation (audio in, audio out) + +Stream audio to a Speechify voice agent in real time and save its spoken reply to a WAV file, with the live transcript printed to your terminal. + +## Prerequisites + +- A Speechify API key — https://console.speechify.ai/api-keys +- Python 3.9+ + +## Setup + +```bash +cp .env.example .env # then paste your SPEECHIFY_API_KEY +uv sync +``` + +## Run + +```bash +uv run main.py +``` + +Writes `agent_reply.wav` (the agent's spoken response) and prints the transcript. + +## What it does + +- Creates a throwaway voice agent, then opens a realtime conversation with `client.agent.create_conversation(...)`. +- Connects with `speechify.realtime` and streams `fixtures/caller.wav` to the agent via `session.send_audio()` — raw PCM16 bytes, no audio device involved. +- Saves the agent's audio from `session.output_audio()` to `agent_reply.wav` and prints transcript updates from `session.on_text()`. +- Deletes the agent on exit. + +> The SDK exposes the realtime session as raw byte streams — wire `send_audio()` / `output_audio()` to any source/sink (a telephony bridge, a WebSocket, files, your own capture lib). +> `fixtures/caller.wav` was produced from a Speechify sample (`https://speechify.ai/audio/multilingual/en.mp3`) with `ffmpeg -i en.mp3 -ar 48000 -ac 1 -sample_fmt s16 caller.wav`. diff --git a/recipes/voice-agents/python/realtime-conversation/fixtures/caller.wav b/recipes/voice-agents/python/realtime-conversation/fixtures/caller.wav new file mode 100644 index 0000000..12d4bab Binary files /dev/null and b/recipes/voice-agents/python/realtime-conversation/fixtures/caller.wav differ diff --git a/recipes/voice-agents/python/realtime-conversation/main.py b/recipes/voice-agents/python/realtime-conversation/main.py new file mode 100644 index 0000000..1a4bce6 --- /dev/null +++ b/recipes/voice-agents/python/realtime-conversation/main.py @@ -0,0 +1,86 @@ +import asyncio +import os +import wave +from pathlib import Path + +from dotenv import load_dotenv +from speechify import Speechify, realtime + +CALLER_WAV = Path(__file__).parent / "fixtures" / "caller.wav" +REPLY_WAV = "agent_reply.wav" +OUTPUT_SAMPLE_RATE = 48000 # session.output_audio() yields 48 kHz mono PCM16 +FRAME_MS = 20 # stream the caller WAV in 20 ms frames, paced to real time + + +async def converse(client: Speechify, agent_id: str) -> None: + # Open a realtime conversation and connect to the returned endpoint. + conversation = client.agent.create_conversation(agent_id) + session = await realtime.connect_conversation(conversation) + print(f"connected to {session.room_name}") + + # Transcripts (caller + agent) as they stream. + session.on_text( + lambda ev: print(f"[{ev.role}] {ev.text}" + (" (final)" if ev.final else "")) + ) + + # Agent audio out -> WAV file (raw PCM16, no audio device). + reply = wave.open(REPLY_WAV, "wb") + reply.setnchannels(1) + reply.setsampwidth(2) + reply.setframerate(OUTPUT_SAMPLE_RATE) + + async def save_reply() -> None: + async for chunk in session.output_audio(): + reply.writeframes(chunk.data) + + # Caller audio in <- WAV file, streamed in real time. + async def stream_caller() -> None: + wf = wave.open(str(CALLER_WAV), "rb") + rate, channels = wf.getframerate(), wf.getnchannels() + frames_per_chunk = int(rate * FRAME_MS / 1000) + while True: + data = wf.readframes(frames_per_chunk) + if not data: + break + await session.send_audio(data, sample_rate=rate, num_channels=channels) + await asyncio.sleep(FRAME_MS / 1000) + wf.close() + + saver = asyncio.create_task(save_reply()) + await stream_caller() + await asyncio.sleep(5) # let the agent finish replying + await session.disconnect() + try: + await asyncio.wait_for(saver, timeout=3) + except asyncio.TimeoutError: + saver.cancel() + reply.close() + print(f"saved agent reply -> {REPLY_WAV}") + + +def main() -> None: + load_dotenv() + + token = os.environ.get("SPEECHIFY_API_KEY") + if not token: + raise SystemExit("Set SPEECHIFY_API_KEY (copy .env.example to .env).") + + client = Speechify(token=token) + + # Create a throwaway agent to talk to, then clean it up at the end. + agent = client.agent.create( + name="Realtime cookbook demo", + voice_id="sabrina", + first_message="Hi! Thanks for calling. How can I help?", + prompt="You are a friendly support agent. Keep your replies short.", + ) + print(f"created agent {agent.id}") + try: + asyncio.run(converse(client, agent.id)) + finally: + client.agent.delete(agent.id) + print(f"deleted agent {agent.id}") + + +if __name__ == "__main__": + main() diff --git a/recipes/voice-agents/python/realtime-conversation/pyproject.toml b/recipes/voice-agents/python/realtime-conversation/pyproject.toml new file mode 100644 index 0000000..50ec15a --- /dev/null +++ b/recipes/voice-agents/python/realtime-conversation/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "voice-agents-python-realtime-conversation" +version = "0.1.0" +description = "Stream audio to a Speechify voice agent in real time and save its spoken reply." +requires-python = ">=3.9" +dependencies = [ + # Realtime support (speechify.realtime) ships in the SDK's realtime branch until released; + # switch to "speechify-api>=" once published to PyPI. + "speechify-api @ git+https://github.com/SpeechifyInc/speechify-api-sdk-python.git@realtime-connect", + "python-dotenv>=1.0.0", +] diff --git a/recipes/voice-agents/typescript/realtime-conversation/.env.example b/recipes/voice-agents/typescript/realtime-conversation/.env.example new file mode 100644 index 0000000..4cb51e4 --- /dev/null +++ b/recipes/voice-agents/typescript/realtime-conversation/.env.example @@ -0,0 +1 @@ +SPEECHIFY_API_KEY= diff --git a/recipes/voice-agents/typescript/realtime-conversation/README.md b/recipes/voice-agents/typescript/realtime-conversation/README.md new file mode 100644 index 0000000..2590b03 --- /dev/null +++ b/recipes/voice-agents/typescript/realtime-conversation/README.md @@ -0,0 +1,34 @@ +# Voice Agents: real-time conversation (audio in, audio out) + +Stream audio to a Speechify voice agent in real time and save its spoken reply to a WAV file, with the live transcript printed to your terminal. + +## Prerequisites + +- A Speechify API key — https://console.speechify.ai/api-keys +- Node.js 20+ + +## Setup + +```bash +cp .env.example .env # then paste your SPEECHIFY_API_KEY +pnpm install +``` + +## Run + +```bash +pnpm start +``` + +Writes `agent_reply.wav` (the agent's spoken response) and prints the transcript. + +## What it does + +- Creates a throwaway voice agent, then opens a realtime conversation with `client.agent.createConversation(...)`. +- Connects with `@speechify/api/realtime` and streams `fixtures/caller.wav` to the agent via `session.sendAudio()` — raw PCM16 bytes, no audio device involved. +- Saves the agent's audio from `session.outputAudio()` to `agent_reply.wav` and prints transcript updates from `session.onText()`. +- Deletes the agent on exit. + +> The SDK exposes the realtime session as raw byte streams — wire `sendAudio()` / `outputAudio()` to any source/sink (a telephony bridge, a WebSocket, files, your own capture lib). +> `fixtures/caller.wav` was produced from a Speechify sample (`https://speechify.ai/audio/multilingual/en.mp3`) with `ffmpeg -i en.mp3 -ar 48000 -ac 1 -sample_fmt s16 caller.wav`. +> Realtime support ships in the SDK's `realtime-connect` branch until released; this recipe pins that branch via git. Switch to the catalog `@speechify/api` version once it's published to npm. diff --git a/recipes/voice-agents/typescript/realtime-conversation/fixtures/caller.wav b/recipes/voice-agents/typescript/realtime-conversation/fixtures/caller.wav new file mode 100644 index 0000000..12d4bab Binary files /dev/null and b/recipes/voice-agents/typescript/realtime-conversation/fixtures/caller.wav differ diff --git a/recipes/voice-agents/typescript/realtime-conversation/package.json b/recipes/voice-agents/typescript/realtime-conversation/package.json new file mode 100644 index 0000000..7bbee7b --- /dev/null +++ b/recipes/voice-agents/typescript/realtime-conversation/package.json @@ -0,0 +1,20 @@ +{ + "name": "voice-agents-typescript-realtime-conversation", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Stream audio to a Speechify voice agent in real time and save its spoken reply.", + "scripts": { + "start": "tsx src/index.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@speechify/api": "github:SpeechifyInc/speechify-api-sdk-typescript#realtime-connect", + "dotenv": "catalog:" + }, + "devDependencies": { + "@types/node": "catalog:", + "tsx": "catalog:", + "typescript": "catalog:" + } +} diff --git a/recipes/voice-agents/typescript/realtime-conversation/src/index.ts b/recipes/voice-agents/typescript/realtime-conversation/src/index.ts new file mode 100644 index 0000000..2a2135e --- /dev/null +++ b/recipes/voice-agents/typescript/realtime-conversation/src/index.ts @@ -0,0 +1,129 @@ +import "dotenv/config"; + +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { SpeechifyClient } from "@speechify/api"; +import { connectConversation } from "@speechify/api/realtime"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CALLER_WAV = join(HERE, "..", "fixtures", "caller.wav"); +const REPLY_WAV = "agent_reply.wav"; +const OUTPUT_SAMPLE_RATE = 48000; // session.outputAudio() yields 48 kHz mono PCM16 +const FRAME_MS = 20; // stream the caller WAV in 20 ms frames, paced to real time + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +interface Wav { + sampleRate: number; + channels: number; + data: Buffer; +} + +// Minimal RIFF/WAVE reader — walks chunks rather than assuming a 44-byte header. +function readWav(path: string): Wav { + const buf = readFileSync(path); + let sampleRate = OUTPUT_SAMPLE_RATE; + let channels = 1; + let data = Buffer.alloc(0); + let offset = 12; // skip "RIFF""WAVE" + while (offset + 8 <= buf.length) { + const id = buf.toString("ascii", offset, offset + 4); + const size = buf.readUInt32LE(offset + 4); + const body = offset + 8; + if (id === "fmt ") { + channels = buf.readUInt16LE(body + 2); + sampleRate = buf.readUInt32LE(body + 4); + } else if (id === "data") { + data = buf.subarray(body, body + size); + } + offset = body + size + (size % 2); // chunks are word-aligned + } + return { sampleRate, channels, data }; +} + +// Minimal PCM16 WAV writer. +function writeWav(path: string, pcm: Buffer, sampleRate: number, channels: number): void { + const header = Buffer.alloc(44); + header.write("RIFF", 0); + header.writeUInt32LE(36 + pcm.length, 4); + header.write("WAVE", 8); + header.write("fmt ", 12); + header.writeUInt32LE(16, 16); + header.writeUInt16LE(1, 20); // PCM + header.writeUInt16LE(channels, 22); + header.writeUInt32LE(sampleRate, 24); + header.writeUInt32LE(sampleRate * channels * 2, 28); + header.writeUInt16LE(channels * 2, 32); + header.writeUInt16LE(16, 34); // bits per sample + header.write("data", 36); + header.writeUInt32LE(pcm.length, 40); + writeFileSync(path, Buffer.concat([header, pcm])); +} + +async function converse(client: SpeechifyClient, agentId: string): Promise { + // Open a realtime conversation and connect to the returned endpoint. + const conversation = await client.agent.createConversation(agentId); + const session = await connectConversation(conversation); + console.log(`connected to ${session.roomName}`); + + // Transcripts (caller + agent) as they stream. + session.onText((ev) => console.log(`[${ev.role}] ${ev.text}${ev.final ? " (final)" : ""}`)); + + // Agent audio out -> collect raw PCM16 (no audio device), written to WAV at the end. + const replyChunks: Buffer[] = []; + const saver = (async () => { + for await (const chunk of session.outputAudio()) { + replyChunks.push(chunk.data); + } + })(); + + // Caller audio in <- WAV file, streamed in real time in 20 ms frames. + const caller = readWav(CALLER_WAV); + const bytesPerFrame = Math.floor((caller.sampleRate * FRAME_MS) / 1000) * caller.channels * 2; + for (let i = 0; i < caller.data.length; i += bytesPerFrame) { + const frame = caller.data.subarray(i, i + bytesPerFrame); + await session.sendAudio(frame, { sampleRate: caller.sampleRate, channels: caller.channels }); + await sleep(FRAME_MS); + } + + await sleep(5000); // let the agent finish replying + await session.disconnect(); + await saver; // outputAudio() ends when the session disconnects + + writeWav(REPLY_WAV, Buffer.concat(replyChunks), OUTPUT_SAMPLE_RATE, 1); + console.log(`saved agent reply -> ${REPLY_WAV}`); +} + +async function main(): Promise { + const token = process.env.SPEECHIFY_API_KEY; + if (!token) { + throw new Error("Set SPEECHIFY_API_KEY (copy .env.example to .env)."); + } + + const client = new SpeechifyClient({ token }); + + // Create a throwaway agent to talk to, then clean it up at the end. + const agent = await client.agent.create({ + name: "Realtime cookbook demo", + voiceId: "sabrina", + firstMessage: "Hi! Thanks for calling. How can I help?", + prompt: "You are a friendly support agent. Keep your replies short.", + }); + console.log(`created agent ${agent.id}`); + try { + await converse(client, agent.id); + } finally { + await client.agent.delete(agent.id); + console.log(`deleted agent ${agent.id}`); + } +} + +// The realtime transport holds open native handles, so exit explicitly once done. +main() + .then(() => process.exit(0)) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/recipes/voice-agents/typescript/realtime-conversation/tsconfig.json b/recipes/voice-agents/typescript/realtime-conversation/tsconfig.json new file mode 100644 index 0000000..fa0ec53 --- /dev/null +++ b/recipes/voice-agents/typescript/realtime-conversation/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../../../tsconfig.base.json", + "include": ["src"] +}