diff --git a/apps/builder/__tests__/create-webchat-message-action.test.ts b/apps/builder/__tests__/create-webchat-message-action.test.ts index 5b74ad6357..6dc4327eb0 100644 --- a/apps/builder/__tests__/create-webchat-message-action.test.ts +++ b/apps/builder/__tests__/create-webchat-message-action.test.ts @@ -121,7 +121,7 @@ vi.mock("@chatbotx.io/automated-response", () => ({ enqueueHandoffReentry: mockAutomatedResponseEnqueueHandoffReentry, }, isKeywordMediaFileType: (fileType: string) => - ["image", "gif", "video"].includes(fileType), + ["image", "gif", "video", "audio"].includes(fileType), })) vi.mock("@chatbotx.io/business", () => ({ diff --git a/apps/builder/__tests__/keywords-public-api.test.ts b/apps/builder/__tests__/keywords-public-api.test.ts index 83c7d8b6f6..19b709da33 100644 --- a/apps/builder/__tests__/keywords-public-api.test.ts +++ b/apps/builder/__tests__/keywords-public-api.test.ts @@ -328,6 +328,31 @@ describe("keyword conditions on create/update", () => { ).toBe(true) }) + test("create and update accept a containsAudio condition with no values", () => { + const audio = [{ operator: "containsAudio", values: [] }] + expect( + create.input?.safeParse({ + type: "inbound", + conditions: audio, + text: "Hi", + }).success, + ).toBe(true) + expect( + update.input?.safeParse({ id: "1", type: "inbound", conditions: audio }) + .success, + ).toBe(true) + }) + + test("create rejects values on a containsAudio condition", () => { + expect( + create.input?.safeParse({ + type: "inbound", + conditions: [{ operator: "containsAudio", values: ["voice"] }], + text: "Hi", + }).success, + ).toBe(false) + }) + test("create rejects keywords and conditions together, or neither", () => { expect( create.input?.safeParse({ diff --git a/apps/builder/messages/ar.json b/apps/builder/messages/ar.json index 5a13e181f8..d788bd81a4 100644 --- a/apps/builder/messages/ar.json +++ b/apps/builder/messages/ar.json @@ -6192,6 +6192,7 @@ "containsPhone": "الرسالة تحتوي على رقم هاتف", "containsEmail": "الرسالة تحتوي على بريد إلكتروني", "containsImage": "الرسالة تحتوي على صورة", + "containsAudio": "الرسالة تحتوي على صوت", "containsVideo": "الرسالة تحتوي على فيديو", "containsLink": "الرسالة تحتوي على رابط", "containsWholeWord": "الرسالة تحتوي على الكلمة كاملة", diff --git a/apps/builder/messages/az.json b/apps/builder/messages/az.json index d26068b2a4..61564ddc72 100644 --- a/apps/builder/messages/az.json +++ b/apps/builder/messages/az.json @@ -6874,6 +6874,7 @@ "containsPhone": "Mesajda telefon var", "containsEmail": "Mesajda e-poçt var", "containsImage": "Mesajda şəkil var", + "containsAudio": "Mesajda audio var", "containsVideo": "Mesajda video var", "containsLink": "Mesajda keçid var", "containsWholeWord": "Mesajda tam söz var", diff --git a/apps/builder/messages/da.json b/apps/builder/messages/da.json index 3ea1754413..79dbdd92ad 100644 --- a/apps/builder/messages/da.json +++ b/apps/builder/messages/da.json @@ -6192,6 +6192,7 @@ "containsPhone": "Beskeden indeholder telefonnummer", "containsEmail": "Beskeden indeholder e-mail", "containsImage": "Beskeden indeholder billede", + "containsAudio": "Beskeden indeholder lyd", "containsVideo": "Beskeden indeholder video", "containsLink": "Beskeden indeholder link", "containsWholeWord": "Beskeden indeholder hele ordet", diff --git a/apps/builder/messages/de.json b/apps/builder/messages/de.json index 42e71dce11..869709e7b8 100644 --- a/apps/builder/messages/de.json +++ b/apps/builder/messages/de.json @@ -6192,6 +6192,7 @@ "containsPhone": "Nachricht enthält Telefonnummer", "containsEmail": "Nachricht enthält E-Mail", "containsImage": "Nachricht enthält Bild", + "containsAudio": "Nachricht enthält Audio", "containsVideo": "Nachricht enthält Video", "containsLink": "Nachricht enthält Link", "containsWholeWord": "Nachricht enthält ganzes Wort", diff --git a/apps/builder/messages/en.json b/apps/builder/messages/en.json index 54a3cd8589..00648cb76c 100644 --- a/apps/builder/messages/en.json +++ b/apps/builder/messages/en.json @@ -6874,6 +6874,7 @@ "containsPhone": "Message contains phone", "containsEmail": "Message contains email", "containsImage": "Message contains image", + "containsAudio": "Message contains audio", "containsVideo": "Message contains video", "containsLink": "Message contains link", "containsWholeWord": "Message contains whole word", diff --git a/apps/builder/messages/es.json b/apps/builder/messages/es.json index dfcd516437..9d28ec6031 100644 --- a/apps/builder/messages/es.json +++ b/apps/builder/messages/es.json @@ -6192,6 +6192,7 @@ "containsPhone": "El mensaje contiene teléfono", "containsEmail": "El mensaje contiene correo", "containsImage": "El mensaje contiene imagen", + "containsAudio": "El mensaje contiene audio", "containsVideo": "El mensaje contiene video", "containsLink": "El mensaje contiene enlace", "containsWholeWord": "El mensaje contiene la palabra completa", diff --git a/apps/builder/messages/fi.json b/apps/builder/messages/fi.json index 30b4d8ec96..8338e0d0ff 100644 --- a/apps/builder/messages/fi.json +++ b/apps/builder/messages/fi.json @@ -6192,6 +6192,7 @@ "containsPhone": "Viesti sisältää puhelinnumeron", "containsEmail": "Viesti sisältää sähköpostin", "containsImage": "Viesti sisältää kuvan", + "containsAudio": "Viesti sisältää äänen", "containsVideo": "Viesti sisältää videon", "containsLink": "Viesti sisältää linkin", "containsWholeWord": "Viesti sisältää kokonaisen sanan", diff --git a/apps/builder/messages/fr.json b/apps/builder/messages/fr.json index 2c3b5fac6e..3715806e3f 100644 --- a/apps/builder/messages/fr.json +++ b/apps/builder/messages/fr.json @@ -6192,6 +6192,7 @@ "containsPhone": "Le message contient un téléphone", "containsEmail": "Le message contient un e-mail", "containsImage": "Le message contient une image", + "containsAudio": "Le message contient un audio", "containsVideo": "Le message contient une vidéo", "containsLink": "Le message contient un lien", "containsWholeWord": "Le message contient le mot entier", diff --git a/apps/builder/messages/he.json b/apps/builder/messages/he.json index 94724d2fe4..b9173011ae 100644 --- a/apps/builder/messages/he.json +++ b/apps/builder/messages/he.json @@ -3837,6 +3837,7 @@ "containsPhone": "ההודעה מכילה טלפון", "containsEmail": "ההודעה מכילה אימייל", "containsImage": "ההודעה מכילה תמונה", + "containsAudio": "ההודעה מכילה אודיו", "containsVideo": "ההודעה מכילה וידאו", "containsLink": "ההודעה מכילה קישור", "containsWholeWord": "ההודעה מכילה מילה שלמה", diff --git a/apps/builder/messages/id.json b/apps/builder/messages/id.json index a1e76b6046..66e11b8513 100644 --- a/apps/builder/messages/id.json +++ b/apps/builder/messages/id.json @@ -6192,6 +6192,7 @@ "containsPhone": "Pesan berisi nomor telepon", "containsEmail": "Pesan berisi email", "containsImage": "Pesan berisi gambar", + "containsAudio": "Pesan berisi audio", "containsVideo": "Pesan berisi video", "containsLink": "Pesan berisi tautan", "containsWholeWord": "Pesan berisi kata utuh", diff --git a/apps/builder/messages/it.json b/apps/builder/messages/it.json index f68a3d3e9e..6bcb8026d0 100644 --- a/apps/builder/messages/it.json +++ b/apps/builder/messages/it.json @@ -6192,6 +6192,7 @@ "containsPhone": "Il messaggio contiene un telefono", "containsEmail": "Il messaggio contiene un'e-mail", "containsImage": "Il messaggio contiene un'immagine", + "containsAudio": "Il messaggio contiene un audio", "containsVideo": "Il messaggio contiene un video", "containsLink": "Il messaggio contiene un link", "containsWholeWord": "Il messaggio contiene la parola intera", diff --git a/apps/builder/messages/ja.json b/apps/builder/messages/ja.json index 532e56a98a..683934d6dd 100644 --- a/apps/builder/messages/ja.json +++ b/apps/builder/messages/ja.json @@ -6169,6 +6169,7 @@ "containsPhone": "メッセージに電話番号を含む", "containsEmail": "メッセージにメールを含む", "containsImage": "メッセージに画像を含む", + "containsAudio": "メッセージに音声を含む", "containsVideo": "メッセージに動画を含む", "containsLink": "メッセージにリンクを含む", "containsWholeWord": "メッセージに単語として含む", diff --git a/apps/builder/messages/nl.json b/apps/builder/messages/nl.json index 6f8991abae..4bf994c8d2 100644 --- a/apps/builder/messages/nl.json +++ b/apps/builder/messages/nl.json @@ -6192,6 +6192,7 @@ "containsPhone": "Bericht bevat telefoonnummer", "containsEmail": "Bericht bevat e-mail", "containsImage": "Bericht bevat afbeelding", + "containsAudio": "Bericht bevat audio", "containsVideo": "Bericht bevat video", "containsLink": "Bericht bevat link", "containsWholeWord": "Bericht bevat heel woord", diff --git a/apps/builder/messages/pt-BR.json b/apps/builder/messages/pt-BR.json index d6fd96f2b9..01a6e8f259 100644 --- a/apps/builder/messages/pt-BR.json +++ b/apps/builder/messages/pt-BR.json @@ -5825,6 +5825,7 @@ "containsPhone": "A mensagem contém telefone", "containsEmail": "A mensagem contém e-mail", "containsImage": "A mensagem contém imagem", + "containsAudio": "A mensagem contém áudio", "containsVideo": "A mensagem contém vídeo", "containsLink": "A mensagem contém link", "containsWholeWord": "A mensagem contém a palavra inteira", diff --git a/apps/builder/messages/pt-PT.json b/apps/builder/messages/pt-PT.json index 861885a9fc..7db0e68d29 100644 --- a/apps/builder/messages/pt-PT.json +++ b/apps/builder/messages/pt-PT.json @@ -6188,6 +6188,7 @@ "containsPhone": "A mensagem contém telefone", "containsEmail": "A mensagem contém e-mail", "containsImage": "A mensagem contém imagem", + "containsAudio": "A mensagem contém áudio", "containsVideo": "A mensagem contém vídeo", "containsLink": "A mensagem contém ligação", "containsWholeWord": "A mensagem contém a palavra inteira", diff --git a/apps/builder/messages/ro.json b/apps/builder/messages/ro.json index 210a616e31..2082826c7f 100644 --- a/apps/builder/messages/ro.json +++ b/apps/builder/messages/ro.json @@ -6192,6 +6192,7 @@ "containsPhone": "Mesajul conține telefon", "containsEmail": "Mesajul conține e-mail", "containsImage": "Mesajul conține imagine", + "containsAudio": "Mesajul conține audio", "containsVideo": "Mesajul conține video", "containsLink": "Mesajul conține link", "containsWholeWord": "Mesajul conține cuvântul întreg", diff --git a/apps/builder/messages/sv.json b/apps/builder/messages/sv.json index f5723f7580..cef701ebb1 100644 --- a/apps/builder/messages/sv.json +++ b/apps/builder/messages/sv.json @@ -5165,6 +5165,7 @@ "containsPhone": "Meddelandet innehåller telefonnummer", "containsEmail": "Meddelandet innehåller e-post", "containsImage": "Meddelandet innehåller bild", + "containsAudio": "Meddelandet innehåller ljud", "containsVideo": "Meddelandet innehåller video", "containsLink": "Meddelandet innehåller länk", "containsWholeWord": "Meddelandet innehåller hela ordet", diff --git a/apps/builder/messages/tr.json b/apps/builder/messages/tr.json index fd76c7c569..38f966195f 100644 --- a/apps/builder/messages/tr.json +++ b/apps/builder/messages/tr.json @@ -6192,6 +6192,7 @@ "containsPhone": "Mesaj telefon içerir", "containsEmail": "Mesaj e-posta içerir", "containsImage": "Mesaj görsel içerir", + "containsAudio": "Mesaj ses içerir", "containsVideo": "Mesaj video içerir", "containsLink": "Mesaj bağlantı içerir", "containsWholeWord": "Mesaj tam kelimeyi içerir", diff --git a/apps/builder/messages/vi.json b/apps/builder/messages/vi.json index 8f3e82e1ce..3ba05ecab8 100644 --- a/apps/builder/messages/vi.json +++ b/apps/builder/messages/vi.json @@ -5901,6 +5901,7 @@ "containsPhone": "Tin nhắn có số điện thoại", "containsEmail": "Tin nhắn có email", "containsImage": "Tin nhắn có hình ảnh", + "containsAudio": "Tin nhắn có âm thanh", "containsVideo": "Tin nhắn có video", "containsLink": "Tin nhắn có đường link", "containsWholeWord": "Tin nhắn chứa nguyên từ", diff --git a/apps/builder/messages/zh-CN.json b/apps/builder/messages/zh-CN.json index af3aee276e..b68edb86f6 100644 --- a/apps/builder/messages/zh-CN.json +++ b/apps/builder/messages/zh-CN.json @@ -5153,6 +5153,7 @@ "containsPhone": "消息包含电话", "containsEmail": "消息包含邮箱", "containsImage": "消息包含图片", + "containsAudio": "消息包含音频", "containsVideo": "消息包含视频", "containsLink": "消息包含链接", "containsWholeWord": "消息包含完整单词", diff --git a/apps/builder/messages/zh-TW.json b/apps/builder/messages/zh-TW.json index ae15ed6db9..46911182a7 100644 --- a/apps/builder/messages/zh-TW.json +++ b/apps/builder/messages/zh-TW.json @@ -6238,6 +6238,7 @@ "containsPhone": "訊息包含電話", "containsEmail": "訊息包含電子郵件", "containsImage": "訊息包含圖片", + "containsAudio": "訊息包含音訊", "containsVideo": "訊息包含影片", "containsLink": "訊息包含連結", "containsWholeWord": "訊息包含完整單字", diff --git a/apps/builder/src/features/automated-response/api/public.ts b/apps/builder/src/features/automated-response/api/public.ts index 5eef53b5e9..c046ee1c21 100644 --- a/apps/builder/src/features/automated-response/api/public.ts +++ b/apps/builder/src/features/automated-response/api/public.ts @@ -20,7 +20,7 @@ const keywordDirectionDescription = "Keyword direction: inbound matches messages from contacts; outbound matches messages sent by the Page/agent. Defaults to inbound. Keep the same type when reading, updating, disabling, or deleting a rule. This is not post/comment automation; use the channel's comment-automation tools for comments." const keywordConditionsDescription = - 'Contact (inbound) match conditions, ManyChat-style. Every condition must match (AND); inside one condition any value may match (OR), except `notContains`, which matches when the message contains none of its values. Text matching ignores case and Vietnamese accents ("tu van" matches "tư vấn"). Operators that take values: is (whole message equals), similarTo (whole message is close to the value, small typos allowed), contains, containsWholeWord, beginsWith, endsWith, notContains, matchesRegex (RE2 syntax, case-insensitive, max 200 chars). Operators that take no values (send `values: []`): containsPhone, containsEmail, containsLink, containsImage, containsVideo, isThumbsUp (message is only 👍). Max 10 conditions, 50 values each. Send either `conditions` or `keywords`, never both; inbound `keywords` alone are stored as one `contains` condition. Not allowed for outbound rules, which match exact `keywords`.' + 'Contact (inbound) match conditions, ManyChat-style. Every condition must match (AND); inside one condition any value may match (OR), except `notContains`, which matches when the message contains none of its values. Text matching ignores case and Vietnamese accents ("tu van" matches "tư vấn"). Operators that take values: is (whole message equals), similarTo (whole message is close to the value, small typos allowed), contains, containsWholeWord, beginsWith, endsWith, notContains, matchesRegex (RE2 syntax, case-insensitive, max 200 chars). Operators that take no values (send `values: []`): containsPhone, containsEmail, containsLink, containsImage, containsAudio (voice note or audio file), containsVideo, isThumbsUp (message is only 👍). A message that is only audio (no text) matches only rules that have a `containsAudio` condition. Max 10 conditions, 50 values each. Send either `conditions` or `keywords`, never both; inbound `keywords` alone are stored as one `contains` condition. Not allowed for outbound rules, which match exact `keywords`.' const keywordMatchInputRefinement = ( data: { keywords?: string[]; conditions?: unknown[] }, diff --git a/apps/builder/src/features/messages/actions/create-webchat-message.action.ts b/apps/builder/src/features/messages/actions/create-webchat-message.action.ts index 01ab115ce9..ae584ff470 100644 --- a/apps/builder/src/features/messages/actions/create-webchat-message.action.ts +++ b/apps/builder/src/features/messages/actions/create-webchat-message.action.ts @@ -333,8 +333,8 @@ export async function handleCreateWebchatMessage({ ) const promises: Promise[] = [] - // An image/video without text still reaches Keywords (image/video - // conditions); the enqueue only queues it when a keyword rule matches. + // An image/video/audio without text still reaches Keywords (image/video/ + // audio conditions); the enqueue only queues it when a keyword rule matches. const attachmentFileTypes = newMessage.attachments.map( (attachment) => attachment.fileType, ) diff --git a/apps/cli/README.md b/apps/cli/README.md index 8155e07b2f..3f432666fa 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -529,7 +529,7 @@ chatbotx keywords create --conditions # inbound only, instead of # {"operator","values"}: conditions are ANDed, values ORed # operators: is, similarTo, contains, containsWholeWord, # beginsWith, endsWith, notContains, matchesRegex, - # containsPhone|Email|Link|Image|Video, isThumbsUp (no values) + # containsPhone|Email|Link|Image|Audio|Video, isThumbsUp (no values) chatbotx keywords update # [--type --keywords | --conditions --text --flowId --folderId] chatbotx keywords status update --status # [--type] enable/disable without touching other fields chatbotx keywords delete # [--type] diff --git a/apps/worker/__tests__/audio-signature.test.ts b/apps/worker/__tests__/audio-signature.test.ts new file mode 100644 index 0000000000..2f592431a1 --- /dev/null +++ b/apps/worker/__tests__/audio-signature.test.ts @@ -0,0 +1,123 @@ +import { describe, expect, test } from "vitest" +import { + AUDIO_SIGNATURE_BYTES, + audioSniffer, + sniffAudioMimeType, +} from "../src/heavy/handlers/audio-signature" + +const ascii = (text: string): number[] => + [...text].map((char) => char.charCodeAt(0)) + +const bytes = (...parts: number[][]): Uint8Array => + Uint8Array.from(parts.flat()) + +describe("sniffAudioMimeType", () => { + test("recognises an MP4 container (a Messenger voice note) by ftyp", () => { + // The real clip starts 00 00 00 18 'ftyp' 'mp42'. + expect(sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftypmp42")))).toBe( + "audio/mp4", + ) + }) + + test.each([ + ["ID3 tag", bytes(ascii("ID3"), [4, 0, 0, 0, 0, 0, 0x21])], + // MPEG-1 Layer III, 128 kbps, 44.1 kHz. + ["MPEG-1 layer III frame", bytes([0xff, 0xfb, 0x90, 0x00])], + // MPEG-2 Layer III. + ["MPEG-2 layer III frame", bytes([0xff, 0xf3, 0x90, 0x00])], + ])("recognises MP3 by its %s", (_name, head) => { + expect(sniffAudioMimeType(head)).toBe("audio/mpeg") + }) + + test("recognises Ogg, FLAC and EBML (WebM)", () => { + expect(sniffAudioMimeType(bytes(ascii("OggS"), [0, 2]))).toBe("audio/ogg") + expect(sniffAudioMimeType(bytes(ascii("fLaC"), [0, 0]))).toBe("audio/flac") + expect(sniffAudioMimeType(bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f]))).toBe( + "audio/webm", + ) + }) + + test("recognises WAV only when RIFF is followed by WAVE", () => { + expect( + sniffAudioMimeType( + bytes(ascii("RIFF"), [0x24, 0, 0, 0], ascii("WAVEfmt ")), + ), + ).toBe("audio/wav") + // Other RIFF files (AVI video, WebP image) are not audio. + expect( + sniffAudioMimeType( + bytes(ascii("RIFF"), [0x24, 0, 0, 0], ascii("AVI LIST")), + ), + ).toBeNull() + expect( + sniffAudioMimeType( + bytes(ascii("RIFF"), [0x24, 0, 0, 0], ascii("WEBPVP8 ")), + ), + ).toBeNull() + }) + + test("does not mistake raw AAC (ADTS) for MP3", () => { + // 0xFFF1 is an ADTS header: the layer bits are 00. + expect(sniffAudioMimeType(bytes([0xff, 0xf1, 0x50, 0x80]))).toBeNull() + expect(sniffAudioMimeType(bytes([0xff, 0xf9, 0x50, 0x80]))).toBeNull() + }) + + test("rejects MPEG frames with reserved header values", () => { + // Reserved version (01). + expect(sniffAudioMimeType(bytes([0xff, 0xeb, 0x90, 0x00]))).toBeNull() + // Invalid bitrate index (1111). + expect(sniffAudioMimeType(bytes([0xff, 0xfb, 0xf0, 0x00]))).toBeNull() + // Reserved sample rate (11). + expect(sniffAudioMimeType(bytes([0xff, 0xfb, 0x9c, 0x00]))).toBeNull() + }) + + test.each([ + ["HTML", bytes(ascii(""))], + ["PDF", bytes(ascii("%PDF-1.7"))], + ["PNG", bytes([0x89], ascii("PNG"), [0x0d, 0x0a, 0x1a, 0x0a])], + ["JPEG", bytes([0xff, 0xd8, 0xff, 0xe0, 0, 0x10])], + ["ZIP", bytes(ascii("PK"), [3, 4, 0, 0])], + ["empty", new Uint8Array(0)], + ["a single byte", bytes([0xff])], + ])("does not recognise %s", (_name, head) => { + expect(sniffAudioMimeType(head)).toBeNull() + }) + + test("only needs the leading AUDIO_SIGNATURE_BYTES", () => { + expect(AUDIO_SIGNATURE_BYTES).toBeGreaterThanOrEqual(12) + }) + + test.each([ + ["HEIC", "heic"], + ["HEIF sequence", "msf1"], + ["AVIF", "avif"], + ["HEIF", "mif1"], + ])("does not mistake a %s image (ftyp %s) for audio", (_name, brand) => { + expect( + sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftyp"), ascii(brand))), + ).toBeNull() + }) + + test.each([ + "mp42", + "M4A ", + "isom", + "3gp4", + "qt ", + ])("keeps recognising an MP4/QuickTime container with brand %s", (brand) => { + expect( + sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftyp"), ascii(brand))), + ).toBe("audio/mp4") + }) + + test("still recognises a head too short to carry a brand", () => { + expect(sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftyp")))).toBe( + "audio/mp4", + ) + }) + + test("audioSniffer pairs the sniffer with the head length it needs", () => { + expect(audioSniffer.bytes).toBe(AUDIO_SIGNATURE_BYTES) + expect(audioSniffer.sniff).toBe(sniffAudioMimeType) + }) +}) diff --git a/apps/worker/__tests__/bounded-download.test.ts b/apps/worker/__tests__/bounded-download.test.ts index 287bce189e..4c72c4425f 100644 --- a/apps/worker/__tests__/bounded-download.test.ts +++ b/apps/worker/__tests__/bounded-download.test.ts @@ -18,6 +18,7 @@ const { downloadWithByteLimit } = await import( "../src/heavy/handlers/bounded-download" ) const { ExpectedHeavyStepError } = await import("../src/heavy/handlers/errors") +const { audioSniffer } = await import("../src/heavy/handlers/audio-signature") function responseWithBody(props: { body: Uint8Array @@ -218,3 +219,254 @@ describe("downloadWithByteLimit", () => { expect(mocks.kyGet).not.toHaveBeenCalled() }) }) + +describe("downloadWithByteLimit content sniffing", () => { + const audioTypes = new Set(["audio/mp4", "audio/mpeg", "audio/wav"]) + const options = { + allowedMimeTypes: audioTypes, + label: "audio", + maxBytes: 1024, + signal: new AbortController().signal, + sniffer: audioSniffer, + url: "https://cdn.example.com/voice", + } + // `ftyp` after a 4-byte box size, like the Messenger voice clip. + const mp4 = Uint8Array.from([ + 0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6d, 0x70, 0x34, 0x32, 0, 0, 0, 0, + 1, 2, 3, + ]) + const html = new TextEncoder().encode("") + + // A stream that serves `chunks` and counts how many were requested. + function streamOf(chunks: Uint8Array[], contentType?: string) { + const state = { pulled: 0, cancelled: false } + const response = new Response( + new ReadableStream( + { + pull(controller) { + const chunk = chunks[state.pulled] + state.pulled += 1 + if (chunk) { + controller.enqueue(chunk) + } else { + controller.close() + } + }, + cancel() { + state.cancelled = true + }, + }, + { highWaterMark: 0 }, + ), + { headers: contentType ? { "content-type": contentType } : {} }, + ) + return { response, state } + } + + beforeEach(() => { + vi.clearAllMocks() + }) + + test.each([ + ["application/octet-stream"], + ["binary/octet-stream"], + ["application/octet-stream; charset=binary"], + [undefined], + ])("accepts audio served as %s by its bytes", async (contentType) => { + mocks.kyGet.mockResolvedValueOnce(streamOf([mp4], contentType).response) + + const result = await downloadWithByteLimit(options) + + expect(result.contentType).toBe("audio/mp4") + expect(result.rawContentType).toBe(contentType ?? "") + expect(result.buffer).toEqual(Buffer.from(mp4)) + }) + + test("returns every byte, even when the stream arrives in tiny chunks", async () => { + const chunks = [ + mp4.slice(0, 2), + mp4.slice(2, 3), + mp4.slice(3, 9), + mp4.slice(9), + ] + mocks.kyGet.mockResolvedValueOnce( + streamOf(chunks, "application/octet-stream").response, + ) + + const result = await downloadWithByteLimit(options) + + expect(result.contentType).toBe("audio/mp4") + expect(result.buffer).toEqual(Buffer.from(mp4)) + }) + + test("recognises a file shorter than the sniffing window", async () => { + const wav = Uint8Array.from([ + 0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x41, 0x56, 0x45, + ]) + mocks.kyGet.mockResolvedValueOnce( + streamOf([wav], "application/octet-stream").response, + ) + + const result = await downloadWithByteLimit(options) + + expect(result.contentType).toBe("audio/wav") + expect(result.buffer).toEqual(Buffer.from(wav)) + }) + + test("drops a generic file that is not audio after its first chunk", async () => { + // A long body: only the first chunk may be read before giving up. + const chunks = Array.from({ length: 50 }, () => html) + const { response, state } = streamOf(chunks, "application/octet-stream") + mocks.kyGet.mockResolvedValueOnce(response) + + await expect(downloadWithByteLimit(options)).rejects.toThrow( + "Unsupported audio format: application/octet-stream", + ) + + expect(state.cancelled).toBe(true) + expect(state.pulled).toBeLessThanOrEqual(2) + }) + + test("rejects a specific but wrong content type without reading the body", async () => { + const { response, state } = streamOf([mp4], "text/html") + mocks.kyGet.mockResolvedValueOnce(response) + + await expect(downloadWithByteLimit(options)).rejects.toBeInstanceOf( + ExpectedHeavyStepError, + ) + + expect(state.pulled).toBe(0) + // The unread body is released, not left holding the connection open. + expect(state.cancelled).toBe(true) + }) + + test("keeps rejecting a generic type when the caller gives no sniffer", async () => { + const { response, state } = streamOf([mp4], "application/octet-stream") + mocks.kyGet.mockResolvedValueOnce(response) + + await expect( + downloadWithByteLimit({ ...options, sniffer: undefined }), + ).rejects.toThrow("Unsupported audio format") + + expect(state.pulled).toBe(0) + expect(state.cancelled).toBe(true) + }) + + test("rejects when the sniffed type is not allowed", async () => { + const ogg = Uint8Array.from([ + 0x4f, 0x67, 0x67, 0x53, 0, 2, 0, 0, 0, 0, 0, 0, + ]) + mocks.kyGet.mockResolvedValueOnce( + streamOf([ogg], "application/octet-stream").response, + ) + + await expect(downloadWithByteLimit(options)).rejects.toThrow( + "Unsupported audio format", + ) + }) + + test("still enforces the byte limit on a sniffed file", async () => { + const big = new Uint8Array(40) + big.set(mp4) + mocks.kyGet.mockResolvedValueOnce( + streamOf([big.slice(0, 20), big.slice(20)], "application/octet-stream") + .response, + ) + + await expect( + downloadWithByteLimit({ ...options, maxBytes: 30 }), + ).rejects.toBeInstanceOf(ExpectedHeavyStepError) + }) + + test("does not sniff when the content type is already allowed", async () => { + // The bytes are not an MP4, but the declared type is trusted as before. + const { response } = streamOf([Uint8Array.from([1, 2, 3, 4])], "audio/mpeg") + mocks.kyGet.mockResolvedValueOnce(response) + + const result = await downloadWithByteLimit(options) + + expect(result.contentType).toBe("audio/mpeg") + expect(result.buffer).toEqual(Buffer.from([1, 2, 3, 4])) + }) + + test("rejects a HEIC image served as octet-stream and cancels the download", async () => { + const heic = Uint8Array.from([ + 0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, 0, 0, 0, 0, + ]) + const { response, state } = streamOf( + [heic, new Uint8Array(64)], + "application/octet-stream", + ) + mocks.kyGet.mockResolvedValueOnce(response) + + await expect(downloadWithByteLimit(options)).rejects.toThrow( + "Unsupported audio format", + ) + + expect(state.cancelled).toBe(true) + }) + + test("releases the stream when the sniffer throws", async () => { + const { response, state } = streamOf( + [mp4, new Uint8Array(64)], + "application/octet-stream", + ) + mocks.kyGet.mockResolvedValueOnce(response) + + await expect( + downloadWithByteLimit({ + ...options, + sniffer: { + bytes: 16, + sniff: () => { + throw new Error("sniffer bug") + }, + }, + }), + ).rejects.toThrow("sniffer bug") + + expect(state.cancelled).toBe(true) + }) + + test("releases the stream when the sniffed type is not allowed", async () => { + const { response, state } = streamOf( + [mp4, new Uint8Array(64)], + "application/octet-stream", + ) + mocks.kyGet.mockResolvedValueOnce(response) + + await expect( + downloadWithByteLimit({ + ...options, + sniffer: { bytes: 16, sniff: () => "audio/ogg" }, + }), + ).rejects.toThrow("Unsupported audio format") + + expect(state.cancelled).toBe(true) + }) + + test("gives the sniffer as many bytes as it asked for", async () => { + const heads: number[] = [] + const head20 = new Uint8Array(20).fill(7) + mocks.kyGet.mockResolvedValueOnce( + streamOf( + [head20.slice(0, 6), head20.slice(6)], + "application/octet-stream", + ).response, + ) + + const result = await downloadWithByteLimit({ + ...options, + sniffer: { + bytes: 20, + sniff: (head) => { + heads.push(head.length) + return "audio/mp4" + }, + }, + }) + + expect(heads).toEqual([20]) + expect(result.buffer.length).toBe(20) + }) +}) diff --git a/apps/worker/__tests__/integration-worker-boot.test.ts b/apps/worker/__tests__/integration-worker-boot.test.ts index d3174834ba..f743e1ff3e 100644 --- a/apps/worker/__tests__/integration-worker-boot.test.ts +++ b/apps/worker/__tests__/integration-worker-boot.test.ts @@ -114,7 +114,7 @@ vi.mock("@chatbotx.io/automated-response", () => ({ enqueueHandoffReentry: vi.fn(), }, isKeywordMediaFileType: (fileType: string) => - ["image", "gif", "video"].includes(fileType), + ["image", "gif", "video", "audio"].includes(fileType), })) vi.mock("@chatbotx.io/business", () => ({ diff --git a/apps/worker/__tests__/integration-worker-incoming-message.test.ts b/apps/worker/__tests__/integration-worker-incoming-message.test.ts index 325e227a34..f2e41976e3 100644 --- a/apps/worker/__tests__/integration-worker-incoming-message.test.ts +++ b/apps/worker/__tests__/integration-worker-incoming-message.test.ts @@ -300,7 +300,7 @@ vi.mock("@chatbotx.io/automated-response", () => ({ enqueueHandoffReentry: vi.fn().mockResolvedValue(undefined), }, isKeywordMediaFileType: (fileType: string) => - ["image", "gif", "video"].includes(fileType), + ["image", "gif", "video", "audio"].includes(fileType), })) vi.mock("@chatbotx.io/database/client", () => ({ @@ -1231,6 +1231,58 @@ describe("integration worker — conversation routing (thread control)", () => { expect(mockAutomatedResponseEnqueue).toHaveBeenCalledTimes(1) }) + describe("a voice note with no text", () => { + const audioMessage = { + ...fakeCreatedMessage, + text: null, + attachments: [ + { + fileType: "audio", + mimeType: "audio/ogg", + originPath: "public/space/ws-1/conversations/conv-1/voice-note", + size: 1024, + }, + ], + } + + beforeEach(async () => { + // A message with attachments is saved through + // `createOrUpdateWithAttachments`, which returns `{ result, isNew }`. + const repository = await mockCreateMessageRepository() + repository.createOrUpdateWithAttachments.mockResolvedValue({ + result: audioMessage, + isNew: true, + }) + }) + + test("reaches the Keywords route as keyword media and is enqueued with its file type", async () => { + mockRunChannelHandler.mockResolvedValue({ + ...parsedMessage({ delivery: "owner" }), + message: { + ...parsedMessage().message, + text: undefined, + attachments: audioMessage.attachments, + }, + }) + + await runJob(incomingJob) + + expect(mockResolveIncomingTextRouting).toHaveBeenCalledWith( + expect.objectContaining({ + hasActionableInput: true, + hasText: false, + hasKeywordMedia: true, + }), + ) + expect(mockAutomatedResponseEnqueue).toHaveBeenCalledWith( + expect.objectContaining({ + messageId: "msg-created", + attachmentFileTypes: ["audio"], + }), + ) + }) + }) + test("an incomingMessageEcho still on the integration queue (drain) is ingested like an incoming message", async () => { mockRunChannelHandler.mockResolvedValue(parsedMessage()) diff --git a/apps/worker/__tests__/speech-to-text-handler.test.ts b/apps/worker/__tests__/speech-to-text-handler.test.ts index 3f56139eec..431e1dd392 100644 --- a/apps/worker/__tests__/speech-to-text-handler.test.ts +++ b/apps/worker/__tests__/speech-to-text-handler.test.ts @@ -221,10 +221,45 @@ describe("speechToTextOutput", () => { expect(mocks.transcribeWithMuse).not.toHaveBeenCalled() }) + test("lets the OpenAI path name a generic content type by the file's bytes", async () => { + const transcription = vi.fn(() => "whisper-model") + mocks.aiFindBy.mockResolvedValue({ auth: {} }) + mocks.getAIModel.mockReturnValue({ transcription }) + mocks.transcribe.mockResolvedValue({ text: "hello" }) + + await run( + AISpeechToTextDefaultFn({ + inputFieldId: "field-in", + outputFieldId: "field-out", + }), + ) + + const [options] = mocks.downloadWithByteLimit.mock.calls[0] ?? [] + expect(options.sniffer.bytes).toBeGreaterThanOrEqual(12) + // The sniffer is the real one: an MP4 voice note is audio/mp4, HTML is not. + const mp4 = Uint8Array.from([ + 0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x6d, 0x70, 0x34, 0x32, + ]) + expect(options.sniffer.sniff(mp4)).toBe("audio/mp4") + expect( + options.sniffer.sniff(new TextEncoder().encode("")), + ).toBeNull() + // Every type it can return is one the allowlist accepts. + expect(options.allowedMimeTypes.has("audio/mp4")).toBe(true) + }) + + test("does not ask Muse for sniffing: it inspects the bytes itself", async () => { + await run() + + const [options] = mocks.downloadWithByteLimit.mock.calls[0] ?? [] + expect(options.sniffer).toBeUndefined() + expect(options.allowedMimeTypes).toBeUndefined() + }) + test("keeps the OpenAI content-type allowlist", async () => { mocks.aiFindBy.mockResolvedValue({ auth: {} }) mocks.getAIModel.mockReturnValue({ transcription: vi.fn() }) - serveAudio(makeWav(), "binary/octet-stream") + serveAudio(makeWav(), "text/html") const error = await captureError( run( diff --git a/apps/worker/src/heavy/handlers/audio-signature.ts b/apps/worker/src/heavy/handlers/audio-signature.ts new file mode 100644 index 0000000000..4150c7c66d --- /dev/null +++ b/apps/worker/src/heavy/handlers/audio-signature.ts @@ -0,0 +1,117 @@ +/** + * Recognises an audio container from the first bytes of a file, for downloads + * whose `content-type` says nothing (`application/octet-stream`, no header). + * + * It only chooses WHICH audio format a file is; every type returned is one of + * the speech-to-text allowlist's. A file it does not recognise is `null`, never + * a guess, so the caller keeps rejecting it. + */ + +/** How many leading bytes `sniffAudioMimeType` may look at. */ +export const AUDIO_SIGNATURE_BYTES = 16 + +const MPEG_FRAME_SYNC = 0xff +// The second byte starts with three more sync bits (0b111xxxxx). +const MPEG_SECOND_BYTE_MIN = 0xe0 +const MPEG_RESERVED_VERSION = 1 +const MPEG_RESERVED_LAYER = 0 +const MPEG_INVALID_BITRATE_INDEX = 15 +const MPEG_INVALID_SAMPLE_RATE_INDEX = 3 + +const matchesAscii = ( + head: Uint8Array, + offset: number, + text: string, +): boolean => { + if (head.length < offset + text.length) { + return false + } + for (let index = 0; index < text.length; index++) { + if (head[offset + index] !== text.charCodeAt(index)) { + return false + } + } + return true +} + +// An MPEG audio frame header: 11 sync bits, then a version and a layer that +// are not the reserved values. A layer of 0 is ADTS (raw AAC), which shares the +// sync bits but is not MP3, so it is left unrecognised. Plain arithmetic keeps +// the bit fields readable without bitwise operators (lint forbids them). +const isMpegAudioFrame = (head: Uint8Array): boolean => { + const first = head[0] + const second = head[1] + if ( + first !== MPEG_FRAME_SYNC || + second === undefined || + second < MPEG_SECOND_BYTE_MIN + ) { + return false + } + const version = Math.floor(second / 8) % 4 + const layer = Math.floor(second / 2) % 4 + if (version === MPEG_RESERVED_VERSION || layer === MPEG_RESERVED_LAYER) { + return false + } + const third = head[2] + if (third === undefined) { + return true + } + const bitrateIndex = Math.floor(third / 16) + const sampleRateIndex = Math.floor(third / 4) % 4 + return ( + bitrateIndex !== MPEG_INVALID_BITRATE_INDEX && + sampleRateIndex !== MPEG_INVALID_SAMPLE_RATE_INDEX + ) +} + +const isEbml = (head: Uint8Array): boolean => + head[0] === 0x1a && head[1] === 0x45 && head[2] === 0xdf && head[3] === 0xa3 + +// `ftyp` brands of still images (HEIF/HEIC/AVIF). They share the box layout with +// MP4 audio but are never audio. Video brands are NOT listed: MP4 audio and +// video share `isom`/`mp42`, so the header cannot tell them apart. +const IMAGE_FTYP_BRANDS: ReadonlySet = new Set([ + "heic", + "heix", + "heim", + "heis", + "hevc", + "hevx", + "mif1", + "msf1", + "avif", + "avis", +]) + +export const sniffAudioMimeType = (head: Uint8Array): string | null => { + // `ftyp` sits after a 4-byte box size. It also opens MP4 video and MOV, so + // the result means "a container speech-to-text can read", not "audio only": + // the transcription provider still has the final say on the content. + if (matchesAscii(head, 4, "ftyp")) { + const brand = String.fromCharCode(...head.subarray(8, 12)) + return IMAGE_FTYP_BRANDS.has(brand) ? null : "audio/mp4" + } + if (matchesAscii(head, 0, "ID3") || isMpegAudioFrame(head)) { + return "audio/mpeg" + } + if (matchesAscii(head, 0, "OggS")) { + return "audio/ogg" + } + if (matchesAscii(head, 0, "RIFF") && matchesAscii(head, 8, "WAVE")) { + return "audio/wav" + } + if (matchesAscii(head, 0, "fLaC")) { + return "audio/flac" + } + if (isEbml(head)) { + return "audio/webm" + } + return null +} + +/** `sniffAudioMimeType` with the head length it needs, for `downloadWithByteLimit`. */ +export const audioSniffer = { + bytes: AUDIO_SIGNATURE_BYTES, + sniff: sniffAudioMimeType, +} diff --git a/apps/worker/src/heavy/handlers/bounded-download.ts b/apps/worker/src/heavy/handlers/bounded-download.ts index 76294ac4e8..d7b0b96653 100644 --- a/apps/worker/src/heavy/handlers/bounded-download.ts +++ b/apps/worker/src/heavy/handlers/bounded-download.ts @@ -8,8 +8,34 @@ import { ExpectedHeavyStepError } from "./errors" const MAX_REDIRECTS = 5 +// Storage often serves a file with one of these when it knows nothing about +// it; the bytes are then the only evidence of what the file is. +const GENERIC_CONTENT_TYPES: ReadonlySet = new Set([ + "application/octet-stream", + "binary/octet-stream", + "", +]) + +/** + * Names a file from its first bytes. `bytes` and `sniff` travel together so a + * sniffer that needs a longer head can never be handed a truncated one. + */ +export type ContentSniffer = { + /** How many leading bytes `sniff` is given. */ + bytes: number + /** Returns the file's MIME type, or `null` to reject it. */ + sniff: (head: Uint8Array) => string | null +} + type DownloadWithByteLimitOptions = { allowedMimeTypes?: ReadonlySet + /** + * Used only when `content-type` is generic (see `GENERIC_CONTENT_TYPES`) and + * not in `allowedMimeTypes`; a specific but wrong type such as `text/html` is + * still rejected. When it returns `null` the file is rejected and the rest of + * the body is never downloaded. + */ + sniffer?: ContentSniffer label: string maxBytes: number signal: AbortSignal @@ -57,11 +83,99 @@ async function assertSafeDownloadUrl( } } +type SniffedBody = { mimeType: string; response: Response } + +function concatChunks(chunks: Uint8Array[], total: number): Uint8Array { + const joined = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + joined.set(chunk, offset) + offset += chunk.byteLength + } + return joined +} + +/** + * Reads just enough of the body to name the file, then hands the body back + * intact (the bytes read are replayed first). A file `sniff` does not + * recognise is abandoned right here: the stream is cancelled, so a large + * non-audio download costs a few KB rather than `maxBytes`. + */ +async function sniffBody( + response: Response, + { bytes: sniffBytes, sniff }: ContentSniffer, +): Promise { + if (!response.body) { + return null + } + const reader = response.body.getReader() + const peeked: Uint8Array[] = [] + let peekedBytes = 0 + let finished = false + // A stream may hand over fewer bytes than asked for at a time. + while (peekedBytes < sniffBytes && !finished) { + const { done, value } = await reader.read() + if (done) { + finished = true + } else if (value) { + peeked.push(value) + peekedBytes += value.byteLength + } + } + + const head = concatChunks(peeked, peekedBytes).subarray(0, sniffBytes) + let mimeType: string | null = null + try { + mimeType = sniff(head) + } finally { + // Also when `sniff` throws: the stream is healthy, so it must be released. + if (!mimeType) { + await reader.cancel().catch(() => undefined) + } + } + if (!mimeType) { + return null + } + + const replay = new ReadableStream({ + start(controller) { + for (const chunk of peeked) { + controller.enqueue(chunk) + } + if (finished) { + controller.close() + } + }, + async pull(controller) { + if (finished) { + return + } + const { done, value } = await reader.read() + if (done) { + controller.close() + } else if (value) { + controller.enqueue(value) + } + }, + cancel: (reason) => reader.cancel(reason), + }) + return { mimeType, response: new Response(replay) } +} + +/** + * Releases the connection of a body that will not be read. A sniffer that + * returned `null` has already cancelled its stream, so callers pass `null`. + */ +async function discardUnread(response: Response | null): Promise { + await response?.body?.cancel().catch(() => undefined) +} + export async function downloadWithByteLimit({ allowedMimeTypes, label, maxBytes, signal, + sniffer, timeout, url, }: DownloadWithByteLimitOptions): Promise { @@ -102,15 +216,23 @@ export async function downloadWithByteLimit({ assertContentLengthWithinLimit(response, label, maxBytes) const rawContentType = response.headers.get("content-type") ?? "" - const contentType = rawContentType.split(";")[0]?.trim() ?? "" + let contentType = rawContentType.split(";")[0]?.trim() ?? "" + let body = response if (allowedMimeTypes && !allowedMimeTypes.has(contentType)) { - throw new ExpectedHeavyStepError( - `Unsupported ${label} format: ${rawContentType || "unknown"}`, - ) + const canSniff = sniffer && GENERIC_CONTENT_TYPES.has(contentType) + const sniffed = canSniff ? await sniffBody(response, sniffer) : null + if (!(sniffed && allowedMimeTypes.has(sniffed.mimeType))) { + await discardUnread(sniffed?.response ?? (canSniff ? null : response)) + throw new ExpectedHeavyStepError( + `Unsupported ${label} format: ${rawContentType || "unknown"}`, + ) + } + contentType = sniffed.mimeType + body = sniffed.response } const buffer = await readBodyWithLimit( - response, + body, maxBytes, (limit) => new ExpectedHeavyStepError( diff --git a/apps/worker/src/heavy/handlers/speech-to-text.ts b/apps/worker/src/heavy/handlers/speech-to-text.ts index c584f1d715..86516f1cd1 100644 --- a/apps/worker/src/heavy/handlers/speech-to-text.ts +++ b/apps/worker/src/heavy/handlers/speech-to-text.ts @@ -14,6 +14,7 @@ import type { HeavyStepComputeProps } from "../../integration/handlers/flow-util import { readCustomFieldValue } from "../../integration/utils/contact" import { logger } from "../../lib/logger" import { getSpeechToTextAttemptTimeoutMs } from "../timeouts" +import { audioSniffer } from "./audio-signature" import { downloadWithByteLimit } from "./bounded-download" import { ExpectedHeavyStepError } from "./errors" @@ -72,6 +73,10 @@ async function transcribeOpenAI({ label: "audio", maxBytes: env.HEAVY_MAX_AUDIO_BYTES, signal, + // Storage may serve a voice note as `application/octet-stream`; the bytes + // then say what it is, and a file that is not audio is dropped after its + // first few bytes. + sniffer: audioSniffer, url: audioUrl, }) diff --git a/apps/worker/src/integration/routing.ts b/apps/worker/src/integration/routing.ts index 4248059f74..804e515e92 100644 --- a/apps/worker/src/integration/routing.ts +++ b/apps/worker/src/integration/routing.ts @@ -38,7 +38,7 @@ export async function resolveIncomingTextRouting(props: { hasActionableInput: boolean // Automated (AI) responses stay text-driven only… hasText: boolean - // …except an image/video, which may still match a Keyword condition. + // …except an image/video/audio, which may still match a Keyword condition. hasKeywordMedia?: boolean isConversationActive: (conversation: ConversationModel) => Promise }): Promise { diff --git a/packages/automated-response/__tests__/dispatch-reply-contact-activity.test.ts b/packages/automated-response/__tests__/dispatch-reply-contact-activity.test.ts index f2bfcdd3e6..e06015189f 100644 --- a/packages/automated-response/__tests__/dispatch-reply-contact-activity.test.ts +++ b/packages/automated-response/__tests__/dispatch-reply-contact-activity.test.ts @@ -164,6 +164,87 @@ describe("dispatchAutomatedResponseReply contact activity", () => { expect(mocks.recordKeywordMatch).not.toHaveBeenCalled() }) + describe("audio-only message", () => { + const audioRule = (overrides: Record = {}) => + rule({ + keywords: [], + conditions: [{ operator: "containsAudio", values: [] }], + flowId: "flow-1", + text: null, + ...overrides, + }) + + const dispatchAudio = (rules: ReturnType[]) => + dispatchAutomatedResponseReply({ + conversation, + contactInbox, + messageId: createId(), + text: "", + attachmentFileTypes: ["audio"], + rules: rules as never, + triggerType: "contact_message_in", + startedByContactMessage: true, + }) + + test("sends the flow with an empty recorded text and automated_response tracking", async () => { + expect(await dispatchAudio([audioRule()])).toBe(true) + + expect(mocks.recordKeywordMatch).toHaveBeenCalledWith( + expect.objectContaining({ automatedResponseId: "ar-1", text: "" }), + ) + expect(mocks.integrationAdd).toHaveBeenCalledTimes(1) + expect(mocks.integrationAdd.mock.calls[0]?.[1]).toMatchObject({ + type: "sendFlow", + data: expect.objectContaining({ + flowId: "flow-1", + startedByContactMessage: true, + trackingContext: expect.objectContaining({ + aiProvider: "none", + }), + }), + }) + }) + + test("is not handled when the flow has no published version", async () => { + mocks.findFlow.mockResolvedValue(null) + + expect(await dispatchAudio([audioRule()])).toBe(false) + + expect(mocks.recordKeywordMatch).not.toHaveBeenCalled() + expect(mocks.integrationAdd).not.toHaveBeenCalled() + }) + + test("is ignored by a rule that only negates text", async () => { + expect( + await dispatchAudio([ + audioRule({ + conditions: [{ operator: "notContains", values: ["stop"] }], + }), + ]), + ).toBe(false) + + expect(mocks.integrationAdd).not.toHaveBeenCalled() + }) + + test("the oldest matching rule wins, and one that cannot send falls through", async () => { + mocks.findFlow + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ id: "flow-2" }) + + expect( + await dispatchAudio([ + audioRule({ id: "ar-dead", flowId: "flow-gone" }), + audioRule({ id: "ar-live", flowId: "flow-2" }), + ]), + ).toBe(true) + + expect(mocks.recordKeywordMatch).toHaveBeenCalledTimes(1) + expect( + mocks.recordKeywordMatch.mock.calls[0]?.[0].automatedResponseId, + ).toBe("ar-live") + }) + }) + test("sends nothing when recording fails, so the job retries cleanly", async () => { mocks.recordKeywordMatch.mockRejectedValue(new Error("db down")) diff --git a/packages/automated-response/__tests__/enqueue-message.test.ts b/packages/automated-response/__tests__/enqueue-message.test.ts index 7da2825e5a..e5384e95a8 100644 --- a/packages/automated-response/__tests__/enqueue-message.test.ts +++ b/packages/automated-response/__tests__/enqueue-message.test.ts @@ -320,4 +320,113 @@ describe("enqueueMessage", () => { expect(mockAiAgentQueueAdd).not.toHaveBeenCalled() }) }) + + describe("audio without text", () => { + const audioProps = { + ...enqueueProps, + messageText: "", + attachmentFileTypes: ["audio" as const], + } + + test("is queued when an inbound audio condition matches it", async () => { + mockWorkspaceFindById.mockResolvedValue({}) + mockAutomatedResponseGetAll.mockResolvedValue([ + { + type: "inbound", + keywords: [], + conditions: [{ operator: "containsAudio", values: [] }], + }, + ]) + + await expect(enqueueMessage(audioProps)).resolves.toBe(true) + + expect(mockAiAgentQueueAdd).toHaveBeenCalledTimes(1) + expect(mockSimpleQueueEnqueue).toHaveBeenCalledWith( + queueKey, + "message-1", + expect.any(Number), + ) + }) + + test("is not queued when only image/video rules exist", async () => { + mockWorkspaceFindById.mockResolvedValue({}) + mockAutomatedResponseGetAll.mockResolvedValue([ + { + type: "inbound", + keywords: [], + conditions: [{ operator: "containsImage", values: [] }], + }, + { + type: "inbound", + keywords: [], + conditions: [{ operator: "containsVideo", values: [] }], + }, + ]) + + await expect(enqueueMessage(audioProps)).resolves.toBe(false) + + expect(mockAiAgentQueueAdd).not.toHaveBeenCalled() + expect(mockSimpleQueueEnqueue).not.toHaveBeenCalled() + }) + + test("is not queued by a rule that only negates text", async () => { + mockWorkspaceFindById.mockResolvedValue({}) + mockAutomatedResponseGetAll.mockResolvedValue([ + { + type: "inbound", + keywords: [], + conditions: [{ operator: "notContains", values: ["stop"] }], + }, + ]) + + await expect(enqueueMessage(audioProps)).resolves.toBe(false) + + expect(mockAiAgentQueueAdd).not.toHaveBeenCalled() + }) + + test("loads the keyword rules once when smart delay is on", async () => { + mockWorkspaceFindById.mockResolvedValue({ smartResponseDelaySeconds: 30 }) + mockAutomatedResponseGetAll.mockResolvedValue([ + { + type: "inbound", + keywords: [], + conditions: [{ operator: "containsAudio", values: [] }], + }, + ]) + + await expect(enqueueMessage(audioProps)).resolves.toBe(true) + + // The media-only gate and the smart-delay check share one lookup. + expect(mockAutomatedResponseGetAll).toHaveBeenCalledTimes(1) + // A matching keyword keeps the v1 env timing, not the 30 s smart delay. + expect(mockAiAgentQueueAdd).toHaveBeenCalledWith( + expect.any(String), + expect.any(Object), + expect.objectContaining({ delay: 2000 }), + ) + }) + + test("is not queued by an outbound rule", async () => { + mockWorkspaceFindById.mockResolvedValue({}) + mockAutomatedResponseGetAll.mockResolvedValue([ + { + type: "outbound", + keywords: [], + conditions: [{ operator: "containsAudio", values: [] }], + }, + ]) + + await expect(enqueueMessage(audioProps)).resolves.toBe(false) + + expect(mockAiAgentQueueAdd).not.toHaveBeenCalled() + }) + + test("is not queued when the keyword lookup fails", async () => { + mockAutomatedResponseGetAll.mockRejectedValue(new Error("redis down")) + + await expect(enqueueMessage(audioProps)).resolves.toBe(false) + + expect(mockAiAgentQueueAdd).not.toHaveBeenCalled() + }) + }) }) diff --git a/packages/automated-response/__tests__/keyword-match.test.ts b/packages/automated-response/__tests__/keyword-match.test.ts index 5c99de5c89..a37575e428 100644 --- a/packages/automated-response/__tests__/keyword-match.test.ts +++ b/packages/automated-response/__tests__/keyword-match.test.ts @@ -1,6 +1,7 @@ import type { KeywordCondition } from "@chatbotx.io/database/partials" import { describe, expect, test } from "vitest" import { + isKeywordMediaFileType, type KeywordMatchInput, keywordConditionMatches, keywordMatchesText, @@ -146,6 +147,169 @@ describe("ruleMatchesMessage", () => { ), ).toBe(false) }) + + describe("audio-only message (no text)", () => { + const audioOnly: KeywordMatchInput = { + text: null, + attachmentFileTypes: ["audio"], + } + const containsAudio: KeywordCondition = { + operator: "containsAudio", + values: [], + } + + test("matches a rule with a containsAudio condition", () => { + expect(ruleMatchesMessage(inboundRule([containsAudio]), audioOnly)).toBe( + true, + ) + }) + + test("does not match rules that only negate text", () => { + expect( + ruleMatchesMessage( + inboundRule([{ operator: "notContains", values: ["stop"] }]), + audioOnly, + ), + ).toBe(false) + expect( + ruleMatchesMessage( + inboundRule([{ operator: "matchesRegex", values: [".*"] }]), + audioOnly, + ), + ).toBe(false) + }) + + test("needs every condition of a rule that has containsAudio", () => { + expect( + ruleMatchesMessage( + inboundRule([ + containsAudio, + { operator: "notContains", values: ["stop"] }, + ]), + audioOnly, + ), + ).toBe(true) + expect( + ruleMatchesMessage( + inboundRule([ + containsAudio, + { operator: "contains", values: ["hi"] }, + ]), + audioOnly, + ), + ).toBe(false) + }) + + test("does not match legacy keyword rules without conditions", () => { + expect( + ruleMatchesMessage( + { keywords: ["hi"], conditions: null, type: "inbound" }, + audioOnly, + ), + ).toBe(false) + }) + + test("a rule with only image/video conditions ignores it", () => { + expect( + ruleMatchesMessage( + inboundRule([{ operator: "containsImage", values: [] }]), + audioOnly, + ), + ).toBe(false) + expect( + ruleMatchesMessage( + inboundRule([{ operator: "containsVideo", values: [] }]), + audioOnly, + ), + ).toBe(false) + }) + + test("audio with text is not audio-only, so text rules still apply", () => { + const audioWithText: KeywordMatchInput = { + text: "nghe nè", + attachmentFileTypes: ["audio"], + } + expect( + ruleMatchesMessage( + inboundRule([{ operator: "notContains", values: ["stop"] }]), + audioWithText, + ), + ).toBe(true) + expect( + ruleMatchesMessage( + inboundRule([{ operator: "contains", values: ["nghe"] }]), + audioWithText, + ), + ).toBe(true) + expect( + ruleMatchesMessage(inboundRule([containsAudio]), audioWithText), + ).toBe(true) + }) + + test("audio plus an image without text is not audio-only", () => { + expect( + ruleMatchesMessage( + inboundRule([{ operator: "notContains", values: ["stop"] }]), + { text: null, attachmentFileTypes: ["audio", "image"] }, + ), + ).toBe(true) + }) + + test("a plain file next to the audio keeps it audio-only", () => { + const audioWithFile: KeywordMatchInput = { + text: null, + attachmentFileTypes: ["audio", "file"], + } + expect( + ruleMatchesMessage( + inboundRule([{ operator: "notContains", values: ["stop"] }]), + audioWithFile, + ), + ).toBe(false) + expect( + ruleMatchesMessage(inboundRule([containsAudio]), audioWithFile), + ).toBe(true) + }) + + test("a whitespace-only caption counts as no text", () => { + const audioWithBlankCaption: KeywordMatchInput = { + text: " \n ", + attachmentFileTypes: ["audio"], + } + expect( + ruleMatchesMessage( + inboundRule([{ operator: "notContains", values: ["stop"] }]), + audioWithBlankCaption, + ), + ).toBe(false) + expect( + ruleMatchesMessage(inboundRule([containsAudio]), audioWithBlankCaption), + ).toBe(true) + }) + }) + + // Pinned on purpose: text-negating operators have always matched an empty + // text, so image/video-only messages still match them. Only audio is opt-in. + test("image/video-only messages still match text-negating rules", () => { + for (const fileType of ["image", "gif", "video"] as const) { + const input: KeywordMatchInput = { + text: null, + attachmentFileTypes: [fileType], + } + expect( + ruleMatchesMessage( + inboundRule([{ operator: "notContains", values: ["stop"] }]), + input, + ), + ).toBe(true) + expect( + ruleMatchesMessage( + inboundRule([{ operator: "matchesRegex", values: [".*"] }]), + input, + ), + ).toBe(true) + } + }) }) describe("keywordConditionMatches", () => { @@ -274,6 +438,33 @@ describe("keywordConditionMatches", () => { expect(matchesCondition(video, withFiles([]))).toBe(false) }) + test("detects audio attachments and keeps the media kinds apart", () => { + const withFiles = ( + attachmentFileTypes: KeywordMatchInput["attachmentFileTypes"], + ) => ({ + text: "nghe nè", + attachmentFileTypes, + }) + const audio: KeywordCondition = { operator: "containsAudio", values: [] } + const image: KeywordCondition = { operator: "containsImage", values: [] } + const video: KeywordCondition = { operator: "containsVideo", values: [] } + expect(matchesCondition(audio, withFiles(["audio"]))).toBe(true) + expect(matchesCondition(audio, withFiles(["image"]))).toBe(false) + expect(matchesCondition(audio, withFiles(["gif"]))).toBe(false) + expect(matchesCondition(audio, withFiles(["video"]))).toBe(false) + expect(matchesCondition(audio, withFiles(["file"]))).toBe(false) + expect(matchesCondition(audio, withFiles([]))).toBe(false) + expect(matchesCondition(image, withFiles(["audio"]))).toBe(false) + expect(matchesCondition(video, withFiles(["audio"]))).toBe(false) + }) + + test("isKeywordMediaFileType accepts image, gif, video and audio only", () => { + for (const fileType of ["image", "gif", "video", "audio"] as const) { + expect(isKeywordMediaFileType(fileType)).toBe(true) + } + expect(isKeywordMediaFileType("file")).toBe(false) + }) + test("isThumbsUp accepts only thumbs-up emoji in any skin tone", () => { const condition: KeywordCondition = { operator: "isThumbsUp", values: [] } expect(matchesCondition(condition, "👍")).toBe(true) diff --git a/packages/automated-response/__tests__/process-messages.test.ts b/packages/automated-response/__tests__/process-messages.test.ts index c32fa58907..ce3eef4244 100644 --- a/packages/automated-response/__tests__/process-messages.test.ts +++ b/packages/automated-response/__tests__/process-messages.test.ts @@ -106,7 +106,6 @@ describe("processPendingMessages", () => { test("does not dispatch attachment, postback, challenge, or quick-reply batches without text", async () => { mockFindManyByIds.mockResolvedValue([ { id: "attachment", text: null, attachmentFileTypes: ["file"] }, - { id: "audio", text: null, attachmentFileTypes: ["audio"] }, { id: "postback", text: "", attachmentFileTypes: [] }, ]) @@ -141,4 +140,26 @@ describe("processPendingMessages", () => { }), ) }) + + test("dispatches an audio-only message with its attachment types", async () => { + mockFindManyByIds.mockResolvedValue([ + { id: "audio", text: null, attachmentFileTypes: ["audio"] }, + ]) + mockDispatchReply.mockResolvedValue(true) + + await expect( + processPendingMessages({ + contactInbox: contactInbox as never, + conversation: conversation as never, + }), + ).resolves.toEqual({ status: "handled" }) + + expect(mockDispatchReply).toHaveBeenCalledWith( + expect.objectContaining({ + messageId: "audio", + text: "", + attachmentFileTypes: ["audio"], + }), + ) + }) }) diff --git a/packages/automated-response/src/dispatch-reply.ts b/packages/automated-response/src/dispatch-reply.ts index dcbad8f2d6..276ba55b3a 100644 --- a/packages/automated-response/src/dispatch-reply.ts +++ b/packages/automated-response/src/dispatch-reply.ts @@ -94,7 +94,7 @@ export const dispatchAutomatedResponseReply = async (props: { AutomatedResponseModel, "id" | "keywords" | "conditions" | "flowId" | "text" | "type" >[] - /** File types attached to the trigger message (image/video conditions). */ + /** File types attached to the trigger message (image/video/audio conditions). */ attachmentFileTypes?: readonly FileType[] triggerType: string /** The rule fired for a contact's inbound message (not an outbound echo). */ diff --git a/packages/automated-response/src/enqueue-message.ts b/packages/automated-response/src/enqueue-message.ts index af52c57bab..0502432e75 100644 --- a/packages/automated-response/src/enqueue-message.ts +++ b/packages/automated-response/src/enqueue-message.ts @@ -25,8 +25,9 @@ const findInboundKeywordRules = async (workspaceId: string) => /** * Queues a contact message for Keywords → AI. Returns false when nothing was - * queued: an image/video without text is queued only when an inbound keyword - * rule matches it, so the AI agent / Global Router never answer a bare photo. + * queued: an image/video/audio without text is queued only when an inbound + * keyword rule matches it, so the AI agent / Global Router never answer a bare + * photo or voice note. */ export const enqueueMessage = async (props: { conversationId: string @@ -44,10 +45,15 @@ export const enqueueMessage = async (props: { } const hasMatchInput = Boolean(matchInput.text) || matchInput.attachmentFileTypes.length > 0 + let rulesLookup: ReturnType | undefined + const loadKeywordRules = () => { + rulesLookup ??= findInboundKeywordRules(props.workspaceId) + return rulesLookup + } if (!matchInput.text && hasMatchInput) { try { - const keywordRules = await findInboundKeywordRules(props.workspaceId) + const keywordRules = await loadKeywordRules() if (!matchesAnyKeywordRule(matchInput, keywordRules)) { return false } @@ -66,9 +72,7 @@ export const enqueueMessage = async (props: { if (isSmartResponseDelayOption(workspaceDelay)) { const [keywordRules, aiAgent, hasGlobalRouter] = await Promise.all([ - hasMatchInput - ? findInboundKeywordRules(props.workspaceId) - : Promise.resolve([]), + hasMatchInput ? loadKeywordRules() : Promise.resolve([]), aiAgentService.findDefault(props.workspaceId), decisionProfileService.hasGlobalRouterEnabled(props.workspaceId), ]) diff --git a/packages/automated-response/src/keyword-match.ts b/packages/automated-response/src/keyword-match.ts index 50ce59a994..b294166af2 100644 --- a/packages/automated-response/src/keyword-match.ts +++ b/packages/automated-response/src/keyword-match.ts @@ -137,10 +137,13 @@ const matchesRegex = (text: string, pattern: string): boolean => const IMAGE_FILE_TYPES: readonly FileType[] = ["image", "gif"] const VIDEO_FILE_TYPES: readonly FileType[] = ["video"] +const AUDIO_FILE_TYPES: readonly FileType[] = ["audio"] -/** Attachment types a keyword condition can match (image/video). */ +/** Attachment types a keyword condition can match (image/video/audio). */ export const isKeywordMediaFileType = (fileType: FileType): boolean => - IMAGE_FILE_TYPES.includes(fileType) || VIDEO_FILE_TYPES.includes(fileType) + IMAGE_FILE_TYPES.includes(fileType) || + VIDEO_FILE_TYPES.includes(fileType) || + AUDIO_FILE_TYPES.includes(fileType) const hasAttachment = ( input: KeywordMatchInput, @@ -189,6 +192,8 @@ export const keywordConditionMatches = ( return hasAttachment(input, IMAGE_FILE_TYPES) case "containsVideo": return hasAttachment(input, VIDEO_FILE_TYPES) + case "containsAudio": + return hasAttachment(input, AUDIO_FILE_TYPES) case "isThumbsUp": return isThumbsUp(rawText) case "matchesRegex": @@ -212,6 +217,27 @@ export const keywordConditionMatches = ( ) } +// A voice note / audio file with no text. Intentionally opt-in: only a rule +// with an explicit `containsAudio` condition may answer it. Text-negating +// operators (`notContains`, an empty-matching `matchesRegex`) also match an +// empty text, so without this a rule like "doesn't contain stop" would start +// answering audio the moment audio reaches Keywords. Image/video-only messages +// keep matching those operators as before. Only keyword media decides it — a +// plain `file` next to the voice note does not make it reachable by text +// rules — and a whitespace-only caption counts as no text. +const isAudioOnlyMessage = (input: KeywordMatchInput): boolean => { + if (input.text?.trim()) { + return false + } + const mediaFileTypes = input.attachmentFileTypes.filter( + isKeywordMediaFileType, + ) + return ( + mediaFileTypes.length > 0 && + mediaFileTypes.every((fileType) => AUDIO_FILE_TYPES.includes(fileType)) + ) +} + /** * Contact (inbound) rules with `conditions` match when every condition does; * rules without them keep the legacy `keywords` + `type` behaviour (Page @@ -222,6 +248,14 @@ export const ruleMatchesMessage = ( input: KeywordMatchInput, ): boolean => { if (rule.conditions && rule.conditions.length > 0) { + if ( + isAudioOnlyMessage(input) && + !rule.conditions.some( + (condition) => condition.operator === "containsAudio", + ) + ) { + return false + } return rule.conditions.every((condition) => keywordConditionMatches(condition, input), ) diff --git a/packages/automated-response/src/process-messages.ts b/packages/automated-response/src/process-messages.ts index 126c0e4854..349f2c556b 100644 --- a/packages/automated-response/src/process-messages.ts +++ b/packages/automated-response/src/process-messages.ts @@ -51,7 +51,7 @@ const getMessagesFromStoreAndProcess = async (props: { return { status: "noEligibleMessage" } } - // Text messages, plus image/video-only messages for the attachment + // Text messages, plus image/video/audio-only messages for the attachment // keyword conditions. const messageRepository = await createMessageRepository() const messages = await messageRepository diff --git a/packages/business/__tests__/automated-response.service.test.ts b/packages/business/__tests__/automated-response.service.test.ts index 8460f74622..952cd4835f 100644 --- a/packages/business/__tests__/automated-response.service.test.ts +++ b/packages/business/__tests__/automated-response.service.test.ts @@ -589,6 +589,20 @@ describe("automatedResponseService keyword conditions", () => { ) }) + test("stores an audio-only rule with its condition and no trigger keywords", async () => { + const conditions = [{ operator: "containsAudio" as const, values: [] }] + + await automatedResponseService.create("workspace-1", { + type: "inbound", + text: "Hi", + conditions, + }) + + expect(mocks.insertValues).toHaveBeenCalledWith( + expect.objectContaining({ keywords: [], conditions }), + ) + }) + test("rejects more plain inbound keywords than one condition allows", async () => { await expect( automatedResponseService.create("workspace-1", { diff --git a/packages/business/__tests__/template-keywords-adapter.test.ts b/packages/business/__tests__/template-keywords-adapter.test.ts index 75f5a0021c..f184864cf0 100644 --- a/packages/business/__tests__/template-keywords-adapter.test.ts +++ b/packages/business/__tests__/template-keywords-adapter.test.ts @@ -119,6 +119,30 @@ describe("keywordsAdapter", () => { expect(values).not.toHaveProperty("keywords") }) + test("installs an audio-only keyword with its condition instead of falling back to keywords", async () => { + mockCreate.mockResolvedValue({ id: "created" }) + const ctx = buildContext() + const audioConditions = [ + { operator: "containsAudio" as const, values: [] }, + ] + + await keywordsAdapter.insert(ctx, [ + { + sourceId: "k1", + type: "inbound", + text: null, + keywords: [], + conditions: audioConditions, + flowId: null, + folderId: null, + }, + ]) + + const [, values] = mockCreate.mock.calls[0] + expect(values).toMatchObject({ conditions: audioConditions }) + expect(ctx.warn).not.toHaveBeenCalled() + }) + test("installs a keyword from a template made before conditions from its plain keywords", async () => { mockCreate.mockResolvedValue({ id: "created" }) const ctx = buildContext() diff --git a/packages/database/__tests__/keyword-condition-schema.test.ts b/packages/database/__tests__/keyword-condition-schema.test.ts index aa3f2c3f15..6d36ffbf28 100644 --- a/packages/database/__tests__/keyword-condition-schema.test.ts +++ b/packages/database/__tests__/keyword-condition-schema.test.ts @@ -23,6 +23,13 @@ describe("keywordConditionSchema", () => { expect(issueMessages({ operator: "contains", values: ["giá"] })).toEqual([]) }) + test("accepts containsAudio with no values and rejects values on it", () => { + expect(issueMessages({ operator: "containsAudio", values: [] })).toEqual([]) + expect(issueMessages({ operator: "containsAudio", values: ["x"] })).toEqual( + ["keywordConditionTakesNoValues"], + ) + }) + test("rejects values on value-less operators", () => { expect(issueMessages({ operator: "containsImage", values: [] })).toEqual([]) expect(issueMessages({ operator: "isThumbsUp", values: ["x"] })).toEqual([ @@ -93,6 +100,7 @@ describe("flattenKeywordConditionValues", () => { flattenKeywordConditionValues([ { operator: "contains", values: ["a", "b"] }, { operator: "containsVideo", values: [] }, + { operator: "containsAudio", values: [] }, { operator: "beginsWith", values: ["b", "c"] }, ]), ).toEqual(["a", "b", "c"]) diff --git a/packages/database/src/partials/automated-response.ts b/packages/database/src/partials/automated-response.ts index 0c1c42934c..d7c3eec5ff 100644 --- a/packages/database/src/partials/automated-response.ts +++ b/packages/database/src/partials/automated-response.ts @@ -51,6 +51,7 @@ export const keywordConditionOperators = z.enum([ "containsPhone", "containsEmail", "containsImage", + "containsAudio", "containsVideo", "containsLink", "containsWholeWord", @@ -68,6 +69,7 @@ export const valuelessKeywordConditionOperators: readonly KeywordConditionOperat "containsPhone", "containsEmail", "containsImage", + "containsAudio", "containsVideo", "containsLink", "isThumbsUp",