feat(keywords): add "message contains audio" keyword condition

apps/builder/__tests__/create-webchat-message-action.test.ts

@@ -121,7 +121,7 @@ vi.mock("@chatbotx.io/automated-response", () => ({
     enqueueHandoffReentry: mockAutomatedResponseEnqueueHandoffReentry,
   },
   isKeywordMediaFileType: (fileType: string) =>
-    ["image", "gif", "video"].includes(fileType),
+    ["image", "gif", "video", "audio"].includes(fileType),
 }))
 
 vi.mock("@chatbotx.io/business", () => ({

apps/builder/__tests__/keywords-public-api.test.ts

@@ -328,6 +328,31 @@ describe("keyword conditions on create/update", () => {
     ).toBe(true)
   })
 
+  test("create and update accept a containsAudio condition with no values", () => {
+    const audio = [{ operator: "containsAudio", values: [] }]
+    expect(
+      create.input?.safeParse({
+        type: "inbound",
+        conditions: audio,
+        text: "Hi",
+      }).success,
+    ).toBe(true)
+    expect(
+      update.input?.safeParse({ id: "1", type: "inbound", conditions: audio })
+        .success,
+    ).toBe(true)
+  })
+
+  test("create rejects values on a containsAudio condition", () => {
+    expect(
+      create.input?.safeParse({
+        type: "inbound",
+        conditions: [{ operator: "containsAudio", values: ["voice"] }],
+        text: "Hi",
+      }).success,
+    ).toBe(false)
+  })
+
   test("create rejects keywords and conditions together, or neither", () => {
     expect(
       create.input?.safeParse({

apps/builder/messages/ar.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "الرسالة تحتوي على رقم هاتف",
         "containsEmail": "الرسالة تحتوي على بريد إلكتروني",
         "containsImage": "الرسالة تحتوي على صورة",
+        "containsAudio": "الرسالة تحتوي على صوت",
         "containsVideo": "الرسالة تحتوي على فيديو",
         "containsLink": "الرسالة تحتوي على رابط",
         "containsWholeWord": "الرسالة تحتوي على الكلمة كاملة",

apps/builder/messages/az.json

@@ -6874,6 +6874,7 @@
         "containsPhone": "Mesajda telefon var",
         "containsEmail": "Mesajda e-poçt var",
         "containsImage": "Mesajda şəkil var",
+        "containsAudio": "Mesajda audio var",
         "containsVideo": "Mesajda video var",
         "containsLink": "Mesajda keçid var",
         "containsWholeWord": "Mesajda tam söz var",

apps/builder/messages/da.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Beskeden indeholder telefonnummer",
         "containsEmail": "Beskeden indeholder e-mail",
         "containsImage": "Beskeden indeholder billede",
+        "containsAudio": "Beskeden indeholder lyd",
         "containsVideo": "Beskeden indeholder video",
         "containsLink": "Beskeden indeholder link",
         "containsWholeWord": "Beskeden indeholder hele ordet",

apps/builder/messages/de.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Nachricht enthält Telefonnummer",
         "containsEmail": "Nachricht enthält E-Mail",
         "containsImage": "Nachricht enthält Bild",
+        "containsAudio": "Nachricht enthält Audio",
         "containsVideo": "Nachricht enthält Video",
         "containsLink": "Nachricht enthält Link",
         "containsWholeWord": "Nachricht enthält ganzes Wort",

apps/builder/messages/en.json

@@ -6874,6 +6874,7 @@
         "containsPhone": "Message contains phone",
         "containsEmail": "Message contains email",
         "containsImage": "Message contains image",
+        "containsAudio": "Message contains audio",
         "containsVideo": "Message contains video",
         "containsLink": "Message contains link",
         "containsWholeWord": "Message contains whole word",

apps/builder/messages/es.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "El mensaje contiene teléfono",
         "containsEmail": "El mensaje contiene correo",
         "containsImage": "El mensaje contiene imagen",
+        "containsAudio": "El mensaje contiene audio",
         "containsVideo": "El mensaje contiene video",
         "containsLink": "El mensaje contiene enlace",
         "containsWholeWord": "El mensaje contiene la palabra completa",

apps/builder/messages/fi.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Viesti sisältää puhelinnumeron",
         "containsEmail": "Viesti sisältää sähköpostin",
         "containsImage": "Viesti sisältää kuvan",
+        "containsAudio": "Viesti sisältää äänen",
         "containsVideo": "Viesti sisältää videon",
         "containsLink": "Viesti sisältää linkin",
         "containsWholeWord": "Viesti sisältää kokonaisen sanan",

apps/builder/messages/fr.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Le message contient un téléphone",
         "containsEmail": "Le message contient un e-mail",
         "containsImage": "Le message contient une image",
+        "containsAudio": "Le message contient un audio",
         "containsVideo": "Le message contient une vidéo",
         "containsLink": "Le message contient un lien",
         "containsWholeWord": "Le message contient le mot entier",

apps/builder/messages/he.json

@@ -3837,6 +3837,7 @@
         "containsPhone": "ההודעה מכילה טלפון",
         "containsEmail": "ההודעה מכילה אימייל",
         "containsImage": "ההודעה מכילה תמונה",
+        "containsAudio": "ההודעה מכילה אודיו",
         "containsVideo": "ההודעה מכילה וידאו",
         "containsLink": "ההודעה מכילה קישור",
         "containsWholeWord": "ההודעה מכילה מילה שלמה",

apps/builder/messages/id.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Pesan berisi nomor telepon",
         "containsEmail": "Pesan berisi email",
         "containsImage": "Pesan berisi gambar",
+        "containsAudio": "Pesan berisi audio",
         "containsVideo": "Pesan berisi video",
         "containsLink": "Pesan berisi tautan",
         "containsWholeWord": "Pesan berisi kata utuh",

apps/builder/messages/it.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Il messaggio contiene un telefono",
         "containsEmail": "Il messaggio contiene un'e-mail",
         "containsImage": "Il messaggio contiene un'immagine",
+        "containsAudio": "Il messaggio contiene un audio",
         "containsVideo": "Il messaggio contiene un video",
         "containsLink": "Il messaggio contiene un link",
         "containsWholeWord": "Il messaggio contiene la parola intera",

apps/builder/messages/ja.json

@@ -6169,6 +6169,7 @@
         "containsPhone": "メッセージに電話番号を含む",
         "containsEmail": "メッセージにメールを含む",
         "containsImage": "メッセージに画像を含む",
+        "containsAudio": "メッセージに音声を含む",
         "containsVideo": "メッセージに動画を含む",
         "containsLink": "メッセージにリンクを含む",
         "containsWholeWord": "メッセージに単語として含む",

apps/builder/messages/nl.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Bericht bevat telefoonnummer",
         "containsEmail": "Bericht bevat e-mail",
         "containsImage": "Bericht bevat afbeelding",
+        "containsAudio": "Bericht bevat audio",
         "containsVideo": "Bericht bevat video",
         "containsLink": "Bericht bevat link",
         "containsWholeWord": "Bericht bevat heel woord",

apps/builder/messages/pt-BR.json

@@ -5825,6 +5825,7 @@
         "containsPhone": "A mensagem contém telefone",
         "containsEmail": "A mensagem contém e-mail",
         "containsImage": "A mensagem contém imagem",
+        "containsAudio": "A mensagem contém áudio",
         "containsVideo": "A mensagem contém vídeo",
         "containsLink": "A mensagem contém link",
         "containsWholeWord": "A mensagem contém a palavra inteira",

apps/builder/messages/pt-PT.json

@@ -6188,6 +6188,7 @@
         "containsPhone": "A mensagem contém telefone",
         "containsEmail": "A mensagem contém e-mail",
         "containsImage": "A mensagem contém imagem",
+        "containsAudio": "A mensagem contém áudio",
         "containsVideo": "A mensagem contém vídeo",
         "containsLink": "A mensagem contém ligação",
         "containsWholeWord": "A mensagem contém a palavra inteira",

apps/builder/messages/ro.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Mesajul conține telefon",
         "containsEmail": "Mesajul conține e-mail",
         "containsImage": "Mesajul conține imagine",
+        "containsAudio": "Mesajul conține audio",
         "containsVideo": "Mesajul conține video",
         "containsLink": "Mesajul conține link",
         "containsWholeWord": "Mesajul conține cuvântul întreg",

apps/builder/messages/sv.json

@@ -5165,6 +5165,7 @@
         "containsPhone": "Meddelandet innehåller telefonnummer",
         "containsEmail": "Meddelandet innehåller e-post",
         "containsImage": "Meddelandet innehåller bild",
+        "containsAudio": "Meddelandet innehåller ljud",
         "containsVideo": "Meddelandet innehåller video",
         "containsLink": "Meddelandet innehåller länk",
         "containsWholeWord": "Meddelandet innehåller hela ordet",

apps/builder/messages/tr.json

@@ -6192,6 +6192,7 @@
         "containsPhone": "Mesaj telefon içerir",
         "containsEmail": "Mesaj e-posta içerir",
         "containsImage": "Mesaj görsel içerir",
+        "containsAudio": "Mesaj ses içerir",
         "containsVideo": "Mesaj video içerir",
         "containsLink": "Mesaj bağlantı içerir",
         "containsWholeWord": "Mesaj tam kelimeyi içerir",

apps/builder/messages/vi.json

@@ -5901,6 +5901,7 @@
         "containsPhone": "Tin nhắn có số điện thoại",
         "containsEmail": "Tin nhắn có email",
         "containsImage": "Tin nhắn có hình ảnh",
+        "containsAudio": "Tin nhắn có âm thanh",
         "containsVideo": "Tin nhắn có video",
         "containsLink": "Tin nhắn có đường link",
         "containsWholeWord": "Tin nhắn chứa nguyên từ",

apps/builder/messages/zh-CN.json

@@ -5153,6 +5153,7 @@
         "containsPhone": "消息包含电话",
         "containsEmail": "消息包含邮箱",
         "containsImage": "消息包含图片",
+        "containsAudio": "消息包含音频",
         "containsVideo": "消息包含视频",
         "containsLink": "消息包含链接",
         "containsWholeWord": "消息包含完整单词",

apps/builder/messages/zh-TW.json

@@ -6238,6 +6238,7 @@
         "containsPhone": "訊息包含電話",
         "containsEmail": "訊息包含電子郵件",
         "containsImage": "訊息包含圖片",
+        "containsAudio": "訊息包含音訊",
         "containsVideo": "訊息包含影片",
         "containsLink": "訊息包含連結",
         "containsWholeWord": "訊息包含完整單字",

apps/builder/src/features/automated-response/api/public.ts

@@ -20,7 +20,7 @@ const keywordDirectionDescription =
   "Keyword direction: inbound matches messages from contacts; outbound matches messages sent by the Page/agent. Defaults to inbound. Keep the same type when reading, updating, disabling, or deleting a rule. This is not post/comment automation; use the channel's comment-automation tools for comments."
 
 const keywordConditionsDescription =
-  'Contact (inbound) match conditions, ManyChat-style. Every condition must match (AND); inside one condition any value may match (OR), except `notContains`, which matches when the message contains none of its values. Text matching ignores case and Vietnamese accents ("tu van" matches "tư vấn"). Operators that take values: is (whole message equals), similarTo (whole message is close to the value, small typos allowed), contains, containsWholeWord, beginsWith, endsWith, notContains, matchesRegex (RE2 syntax, case-insensitive, max 200 chars). Operators that take no values (send `values: []`): containsPhone, containsEmail, containsLink, containsImage, containsVideo, isThumbsUp (message is only 👍). Max 10 conditions, 50 values each. Send either `conditions` or `keywords`, never both; inbound `keywords` alone are stored as one `contains` condition. Not allowed for outbound rules, which match exact `keywords`.'
+  'Contact (inbound) match conditions, ManyChat-style. Every condition must match (AND); inside one condition any value may match (OR), except `notContains`, which matches when the message contains none of its values. Text matching ignores case and Vietnamese accents ("tu van" matches "tư vấn"). Operators that take values: is (whole message equals), similarTo (whole message is close to the value, small typos allowed), contains, containsWholeWord, beginsWith, endsWith, notContains, matchesRegex (RE2 syntax, case-insensitive, max 200 chars). Operators that take no values (send `values: []`): containsPhone, containsEmail, containsLink, containsImage, containsAudio (voice note or audio file), containsVideo, isThumbsUp (message is only 👍). A message that is only audio (no text) matches only rules that have a `containsAudio` condition. Max 10 conditions, 50 values each. Send either `conditions` or `keywords`, never both; inbound `keywords` alone are stored as one `contains` condition. Not allowed for outbound rules, which match exact `keywords`.'
 
 const keywordMatchInputRefinement = (
   data: { keywords?: string[]; conditions?: unknown[] },

apps/builder/src/features/messages/actions/create-webchat-message.action.ts

@@ -333,8 +333,8 @@ export async function handleCreateWebchatMessage({
     )
 
     const promises: Promise<unknown>[] = []
-    // An image/video without text still reaches Keywords (image/video
-    // conditions); the enqueue only queues it when a keyword rule matches.
+    // An image/video/audio without text still reaches Keywords (image/video/
+    // audio conditions); the enqueue only queues it when a keyword rule matches.
     const attachmentFileTypes = newMessage.attachments.map(
       (attachment) => attachment.fileType,
     )

apps/cli/README.md

@@ -529,7 +529,7 @@ chatbotx keywords create --conditions <conditions>   # inbound only, instead of
                                                      # {"operator","values"}: conditions are ANDed, values ORed
                                                      # operators: is, similarTo, contains, containsWholeWord,
                                                      # beginsWith, endsWith, notContains, matchesRegex,
-                                                     # containsPhone|Email|Link|Image|Video, isThumbsUp (no values)
+                                                     # containsPhone|Email|Link|Image|Audio|Video, isThumbsUp (no values)
 chatbotx keywords update <id>                        # [--type --keywords | --conditions --text --flowId --folderId]
 chatbotx keywords status update <id> --status <status>  # [--type]  enable/disable without touching other fields
 chatbotx keywords delete <id>                        # [--type]

apps/worker/__tests__/audio-signature.test.ts

@@ -0,0 +1,123 @@
+import { describe, expect, test } from "vitest"
+import {
+  AUDIO_SIGNATURE_BYTES,
+  audioSniffer,
+  sniffAudioMimeType,
+} from "../src/heavy/handlers/audio-signature"
+
+const ascii = (text: string): number[] =>
+  [...text].map((char) => char.charCodeAt(0))
+
+const bytes = (...parts: number[][]): Uint8Array =>
+  Uint8Array.from(parts.flat())
+
+describe("sniffAudioMimeType", () => {
+  test("recognises an MP4 container (a Messenger voice note) by ftyp", () => {
+    // The real clip starts 00 00 00 18 'ftyp' 'mp42'.
+    expect(sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftypmp42")))).toBe(
+      "audio/mp4",
+    )
+  })
+
+  test.each([
+    ["ID3 tag", bytes(ascii("ID3"), [4, 0, 0, 0, 0, 0, 0x21])],
+    // MPEG-1 Layer III, 128 kbps, 44.1 kHz.
+    ["MPEG-1 layer III frame", bytes([0xff, 0xfb, 0x90, 0x00])],
+    // MPEG-2 Layer III.
+    ["MPEG-2 layer III frame", bytes([0xff, 0xf3, 0x90, 0x00])],
+  ])("recognises MP3 by its %s", (_name, head) => {
+    expect(sniffAudioMimeType(head)).toBe("audio/mpeg")
+  })
+
+  test("recognises Ogg, FLAC and EBML (WebM)", () => {
+    expect(sniffAudioMimeType(bytes(ascii("OggS"), [0, 2]))).toBe("audio/ogg")
+    expect(sniffAudioMimeType(bytes(ascii("fLaC"), [0, 0]))).toBe("audio/flac")
+    expect(sniffAudioMimeType(bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f]))).toBe(
+      "audio/webm",
+    )
+  })
+
+  test("recognises WAV only when RIFF is followed by WAVE", () => {
+    expect(
+      sniffAudioMimeType(
+        bytes(ascii("RIFF"), [0x24, 0, 0, 0], ascii("WAVEfmt ")),
+      ),
+    ).toBe("audio/wav")
+    // Other RIFF files (AVI video, WebP image) are not audio.
+    expect(
+      sniffAudioMimeType(
+        bytes(ascii("RIFF"), [0x24, 0, 0, 0], ascii("AVI LIST")),
+      ),
+    ).toBeNull()
+    expect(
+      sniffAudioMimeType(
+        bytes(ascii("RIFF"), [0x24, 0, 0, 0], ascii("WEBPVP8 ")),
+      ),
+    ).toBeNull()
+  })
+
+  test("does not mistake raw AAC (ADTS) for MP3", () => {
+    // 0xFFF1 is an ADTS header: the layer bits are 00.
+    expect(sniffAudioMimeType(bytes([0xff, 0xf1, 0x50, 0x80]))).toBeNull()
+    expect(sniffAudioMimeType(bytes([0xff, 0xf9, 0x50, 0x80]))).toBeNull()
+  })
+
+  test("rejects MPEG frames with reserved header values", () => {
+    // Reserved version (01).
+    expect(sniffAudioMimeType(bytes([0xff, 0xeb, 0x90, 0x00]))).toBeNull()
+    // Invalid bitrate index (1111).
+    expect(sniffAudioMimeType(bytes([0xff, 0xfb, 0xf0, 0x00]))).toBeNull()
+    // Reserved sample rate (11).
+    expect(sniffAudioMimeType(bytes([0xff, 0xfb, 0x9c, 0x00]))).toBeNull()
+  })
+
+  test.each([
+    ["HTML", bytes(ascii("<!doctype html>"))],
+    ["PDF", bytes(ascii("%PDF-1.7"))],
+    ["PNG", bytes([0x89], ascii("PNG"), [0x0d, 0x0a, 0x1a, 0x0a])],
+    ["JPEG", bytes([0xff, 0xd8, 0xff, 0xe0, 0, 0x10])],
+    ["ZIP", bytes(ascii("PK"), [3, 4, 0, 0])],
+    ["empty", new Uint8Array(0)],
+    ["a single byte", bytes([0xff])],
+  ])("does not recognise %s", (_name, head) => {
+    expect(sniffAudioMimeType(head)).toBeNull()
+  })
+
+  test("only needs the leading AUDIO_SIGNATURE_BYTES", () => {
+    expect(AUDIO_SIGNATURE_BYTES).toBeGreaterThanOrEqual(12)
+  })
+
+  test.each([
+    ["HEIC", "heic"],
+    ["HEIF sequence", "msf1"],
+    ["AVIF", "avif"],
+    ["HEIF", "mif1"],
+  ])("does not mistake a %s image (ftyp %s) for audio", (_name, brand) => {
+    expect(
+      sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftyp"), ascii(brand))),
+    ).toBeNull()
+  })
+
+  test.each([
+    "mp42",
+    "M4A ",
+    "isom",
+    "3gp4",
+    "qt  ",
+  ])("keeps recognising an MP4/QuickTime container with brand %s", (brand) => {
+    expect(
+      sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftyp"), ascii(brand))),
+    ).toBe("audio/mp4")
+  })
+
+  test("still recognises a head too short to carry a brand", () => {
+    expect(sniffAudioMimeType(bytes([0, 0, 0, 0x18], ascii("ftyp")))).toBe(
+      "audio/mp4",
+    )
+  })
+
+  test("audioSniffer pairs the sniffer with the head length it needs", () => {
+    expect(audioSniffer.bytes).toBe(AUDIO_SIGNATURE_BYTES)
+    expect(audioSniffer.sniff).toBe(sniffAudioMimeType)
+  })
+})