From fad411c45cb90ddfe54f8b172064f6504dda3cf0 Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Tue, 10 Nov 2026 14:42:02 -0800 Subject: [PATCH 1/5] Play voice messages inline with bounded normalized audio --- .github/workflows/ci.yml | 5 + AudioMessage.qml | 151 ++++++++++++++++++++++++++++++ AudioPlayback.qml | 36 +++++++ AudioWaveform.mjs | 72 ++++++++++++++ BlipView.qml | 156 ++++++++++++++++++++++++------- audio-metadata.ts | 101 ++++++++++++++++++++ audio-waveform.test.ts | 113 ++++++++++++++++++++++ audio-waveform.ts | 64 +++++++++++++ docs/AUDIO.md | 41 ++++++++ docs/img/inline-audio.png | Bin 0 -> 5162 bytes fetch.ts | 7 ++ scripts/blip-setup | 26 +++++- scripts/test-audio-session-ui.py | 125 +++++++++++++++++++++++++ scripts/test-audio-ui.py | 155 ++++++++++++++++++++++++++++++ tier2.test.ts | 16 ++++ ui.test.ts | 19 ++-- 16 files changed, 1040 insertions(+), 47 deletions(-) create mode 100644 AudioMessage.qml create mode 100644 AudioPlayback.qml create mode 100644 AudioWaveform.mjs create mode 100644 audio-metadata.ts create mode 100644 audio-waveform.test.ts create mode 100644 audio-waveform.ts create mode 100644 docs/AUDIO.md create mode 100644 docs/img/inline-audio.png create mode 100644 scripts/test-audio-session-ui.py create mode 100644 scripts/test-audio-ui.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2439e4b8..0c43e067 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,11 @@ jobs: run: | sudo apt-get install -y -q qml6-module-qtquick-controls qml6-module-qtquick-templates python3 scripts/test-composer-emoji-ui.py + - name: Inline audio playback and captured play intent + run: | + sudo apt-get install -y -q ffmpeg qml6-module-qtmultimedia qml6-module-qtquick-controls qml6-module-qtquick-templates + python3 scripts/test-audio-ui.py + python3 scripts/test-audio-session-ui.py - name: OTP desktop adapter (no desktop needed) run: | python3 -m py_compile otp-desktop.py diff --git a/AudioMessage.qml b/AudioMessage.qml new file mode 100644 index 00000000..90eb868a --- /dev/null +++ b/AudioMessage.qml @@ -0,0 +1,151 @@ +import QtQuick +import QtQuick.Controls +import "AudioWaveform.mjs" as AudioWaveform + +// One parent-owned playing intent selects the active voice message. Each +// item retains its paused position; a conversation reset stops it entirely. +Rectangle { + id: root + property var playback: null + readonly property var player: playback || idle + QtObject { id: idle; property real position: 0; property real duration: 0; property bool seekable: false; property bool ready: false; property bool decodeFailed: false; property bool actuallyPlaying: false; function seek(ms) {} } + property url source: "" + property var waveform: [] + property real durationMs: 0 + property bool playing: false + property bool loading: false + property bool failed: false + property int resetToken: 0 + property bool muted: false + property bool mine: false + property color fillColor: mine ? "#0a84ff" : "#34343a" + property color textColor: "white" + property string fontFamily: "sans-serif" + property int fontSize: 12 + readonly property var bars: AudioWaveform.fitWaveform(waveform, Math.max(1, width - 80)) + readonly property real positionMs: player.position + readonly property real totalDurationMs: player.duration > 0 ? player.duration : durationMs + readonly property real progress: AudioWaveform.playbackFraction(positionMs, totalDurationMs) + readonly property bool actuallyPlaying: player.actuallyPlaying + readonly property bool ready: player.ready + readonly property bool seekable: player.seekable + readonly property bool decodeFailed: player.decodeFailed + readonly property bool controlsFocused: playButton.activeFocus || timeline.activeFocus + signal toggleRequested() + signal playbackEnded() + signal playbackFailed() + signal contextRequested() + signal escapeRequested() + Keys.onEscapePressed: root.escapeRequested() + + implicitWidth: 268 + implicitHeight: 68 + radius: 18 + color: fillColor + Item { + id: playButton + x: 10; y: (root.height - height) / 2 + width: 38; height: 38 + activeFocusOnTab: true + Accessible.role: Accessible.Button + Accessible.name: root.loading ? "Loading audio message" : root.failed || root.decodeFailed ? "Retry audio message" : root.playing ? "Pause audio message" : "Play audio message" + Accessible.description: AudioWaveform.durationLabel(root.totalDurationMs) + Accessible.onPressAction: root.toggleRequested() + Rectangle { anchors.fill: parent; radius: width / 2; color: root.textColor; opacity: playButton.activeFocus ? 0.22 : 0.13 } + // Draw playback symbols geometrically, independent of emoji/font glyphs. + Canvas { + id: playSymbol + anchors.centerIn: parent; width: 20; height: 22 + visible: !root.playing + onPaint: { + var c = getContext("2d") + c.reset(); c.fillStyle = root.textColor + c.beginPath(); c.moveTo(5, 3); c.lineTo(18, 11); c.lineTo(5, 19); c.closePath(); c.fill() + } + Connections { target: root; function onTextColorChanged() { playSymbol.requestPaint() } } + } + Row { + anchors.centerIn: parent; spacing: 4; visible: root.playing + Rectangle { width: 4; height: 17; radius: 1; color: root.textColor } + Rectangle { width: 4; height: 17; radius: 1; color: root.textColor } + } + TapHandler { onTapped: root.toggleRequested() } + HoverHandler { cursorShape: Qt.PointingHandCursor } + Keys.onSpacePressed: root.toggleRequested() + Keys.onReturnPressed: root.toggleRequested() + Keys.onEnterPressed: root.toggleRequested() + } + + Slider { + id: timeline + objectName: "audioTimeline" + x: 58; y: 10 + width: Math.max(24, root.width - x - 14); height: 30 + from: 0; to: Math.max(1, root.totalDurationMs) + value: root.positionMs + stepSize: 1000 + enabled: root.seekable && !root.loading && !root.decodeFailed + activeFocusOnTab: enabled + Accessible.name: "Audio message position" + Accessible.description: (root.bars.length === 0 ? "Waveform unavailable. " : "") + + AudioWaveform.durationLabel(root.positionMs) + " of " + AudioWaveform.durationLabel(root.totalDurationMs) + onMoved: player.seek(Math.round(value)) + background: Item { + x: timeline.leftPadding; y: timeline.topPadding + width: timeline.availableWidth; height: timeline.availableHeight + Row { + anchors.fill: parent + spacing: root.bars.length > 1 ? 2 : 0 + Repeater { + model: root.bars + delegate: Rectangle { + required property int index + required property real modelData + width: Math.max(1, (timeline.availableWidth - Math.max(0, root.bars.length - 1) * 2) / Math.max(1, root.bars.length)) + height: Math.max(3, modelData * timeline.availableHeight) + y: (timeline.availableHeight - height) / 2 + radius: 1.5 + color: root.textColor + opacity: (index + 0.5) / root.bars.length <= root.progress ? 1 : 0.38 + } + } + } + Rectangle { + visible: root.bars.length === 0 + anchors.verticalCenter: parent.verticalCenter + width: parent.width; height: 3; radius: 1.5 + color: root.textColor; opacity: 0.3 + } + Rectangle { + visible: root.bars.length === 0 + anchors.verticalCenter: parent.verticalCenter + width: parent.width * root.progress; height: 3; radius: 1.5 + color: root.textColor + } + } + handle: Rectangle { + x: timeline.leftPadding + timeline.visualPosition * Math.max(0, timeline.availableWidth - width) + y: timeline.topPadding - 1 + width: 2; height: timeline.availableHeight + 2; radius: 1 + visible: timeline.pressed || timeline.activeFocus + color: root.textColor + } + Keys.onLeftPressed: player.seek(Math.max(0, player.position - 5000)) + Keys.onRightPressed: player.seek(Math.min(root.totalDurationMs, player.position + 5000)) + Keys.onSpacePressed: root.toggleRequested() + Keys.onReturnPressed: root.toggleRequested() + Keys.onEnterPressed: root.toggleRequested() + } + Text { + x: timeline.x + 4; y: 42 + width: timeline.width - 8 + text: root.loading ? "Loading…" : root.failed || root.decodeFailed ? "Audio unavailable Β· retry" + : root.totalDurationMs > 0 ? AudioWaveform.durationLabel(Math.max(0, root.totalDurationMs - root.positionMs)) : "Audio message" + textFormat: Text.PlainText + color: root.textColor + opacity: 0.88 + font.family: root.fontFamily + font.pixelSize: root.fontSize + } + TapHandler { acceptedButtons: Qt.RightButton; onTapped: root.contextRequested() } +} diff --git a/AudioPlayback.qml b/AudioPlayback.qml new file mode 100644 index 00000000..17587a49 --- /dev/null +++ b/AudioPlayback.qml @@ -0,0 +1,36 @@ +import QtQuick +import QtMultimedia +Item { + id: root + property url source: "" + property bool playing: false + property bool muted: false + property int resetToken: 0 + readonly property real position: player.position + readonly property real duration: player.duration + readonly property bool seekable: player.seekable + readonly property bool actuallyPlaying: player.playbackState === MediaPlayer.PlayingState + readonly property bool ready: player.mediaStatus === MediaPlayer.LoadedMedia || player.mediaStatus === MediaPlayer.BufferedMedia || player.mediaStatus === MediaPlayer.EndOfMedia + readonly property bool decodeFailed: player.error !== MediaPlayer.NoError + signal playbackEnded() + signal playbackFailed() + function seek(ms) { player.position = ms } + onPlayingChanged: { + if (playing && String(source) !== "") { + if (player.mediaStatus === MediaPlayer.EndOfMedia) player.position = 0 + player.play() + } else player.pause() + } + onResetTokenChanged: player.stop() + Component.onDestruction: player.stop() + MediaPlayer { + id: player + source: root.source + audioOutput: AudioOutput { muted: root.muted } + onMediaStatusChanged: { + if (mediaStatus === MediaPlayer.LoadedMedia && root.playing) play() + else if (mediaStatus === MediaPlayer.EndOfMedia) root.playbackEnded() + } + onErrorOccurred: root.playbackFailed() + } +} diff --git a/AudioWaveform.mjs b/AudioWaveform.mjs new file mode 100644 index 00000000..076ab8aa --- /dev/null +++ b/AudioWaveform.mjs @@ -0,0 +1,72 @@ +// audio-waveform.ts +var AUDIO_WAVEFORM_BARS = 64; +var AUDIO_SAMPLE_RATE = 24000; +var AUDIO_ANALYSIS_SECONDS = 600; +function normalizeWaveform(value) { + if (!Array.isArray(value) || value.length > 96) + return []; + return value.map((v) => typeof v === "number" && Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0); +} +function fitWaveform(value, width) { + const input = normalizeWaveform(value); + if (!input.length || !Number.isFinite(width) || width <= 0) + return []; + const count = Math.min(input.length, Math.max(1, Math.floor(width / 4))); + const output = []; + for (let i = 0;i < count; i++) { + let peak = 0; + for (let j = Math.floor(i * input.length / count);j < Math.floor((i + 1) * input.length / count); j++) + peak = Math.max(peak, input[j]); + output.push(peak); + } + return output; +} +function waveformFromPcm16(bytes, count = AUDIO_WAVEFORM_BARS) { + const samples = Math.floor(bytes.length / 2); + if (!samples || bytes.length > AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 || !Number.isInteger(count) || count < 1 || count > 96) + return []; + count = Math.min(count, samples); + const bins = []; + let largest = 0; + for (let bin = 0;bin < count; bin++) { + const first = Math.floor(bin * samples / count), last = Math.floor((bin + 1) * samples / count); + let sum = 0; + for (let i = first;i < last; i++) { + const unsigned = bytes[2 * i] | bytes[2 * i + 1] << 8; + const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned; + sum += sample * sample; + } + const rms = Math.sqrt(sum / (last - first)); + bins.push(rms); + largest = Math.max(largest, rms); + } + return largest === 0 ? bins : bins.map((v) => Math.round(Math.sqrt(v / largest) * 1000) / 1000); +} +function durationLabel(milliseconds) { + if (!Number.isFinite(milliseconds) || milliseconds < 0) + return "0:00"; + const seconds = Math.floor(Math.min(milliseconds, 86400000) / 1000); + const minutes = Math.floor(seconds / 60), remainder = seconds % 60; + return String(minutes) + ":" + (remainder < 10 ? "0" : "") + String(remainder); +} +function playbackFraction(positionMs, durationMs) { + if (!Number.isFinite(positionMs) || !Number.isFinite(durationMs) || durationMs <= 0) + return 0; + return Math.max(0, Math.min(1, positionMs / durationMs)); +} +function seekPosition(fraction, durationMs) { + if (!Number.isFinite(fraction) || !Number.isFinite(durationMs) || durationMs <= 0) + return 0; + return Math.round(Math.max(0, Math.min(1, fraction)) * durationMs); +} +export { + AUDIO_ANALYSIS_SECONDS, + AUDIO_SAMPLE_RATE, + AUDIO_WAVEFORM_BARS, + durationLabel, + fitWaveform, + normalizeWaveform, + playbackFraction, + seekPosition, + waveformFromPcm16 +}; diff --git a/BlipView.qml b/BlipView.qml index 2b593da6..f60e5af4 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -81,7 +81,8 @@ FocusScope { readonly property color dim: appearance.muted /** An editor owns the keyboard β€” the host's key catcher must stand down. */ readonly property bool editorActive: - emojiPopup.visible || messageMenu.visible || contactMenu.visible || contactReview.opened || composeField.activeFocus || searchField.activeFocus || newField.activeFocus || bubbleFocused + emojiPopup.visible || messageMenu.visible || contactMenu.visible || contactReview.opened || composeField.activeFocus || searchField.activeFocus || newField.activeFocus || bubbleFocused || focusedAudio !== null + property var focusedAudio: null readonly property alias composeEditor: composeField readonly property real contentHeightHint: listContent.implicitHeight /** The view wants keyboard navigation focus back (list mode). */ @@ -878,7 +879,7 @@ FocusScope { /** Chip icon for an attachment's mime type. */ function attachmentIcon(mime) { - var m = String(mime || "") + var m = String(mime || "").toLowerCase() if (m.indexOf("image/") === 0) return "πŸ“·" if (m.indexOf("video/") === 0) return "🎬" if (m.indexOf("audio/") === 0) return "🎀" @@ -1084,38 +1085,81 @@ FocusScope { Qt.callLater(root.pumpAvatar) } } - /** Voice messages play WITHOUT a window. xdg-open handed audio to mpv, - * which opens an empty black video window for a sound file (Fred, - * 2026-09-26). mpv runs with --no-video --force-window=no; the chip toggles: - * a second click on the same file stops it, a click on another stops the - * first. The path is an argument, never interpolated; without mpv the file - * still goes to xdg-open. */ + // Exactly one inline audio message plays. A paused source keeps its + // position; leaving the conversation resets every player and pending intent. + property bool audioMuted: false // synthetic demo/test playback only property string playingAudio: "" - function toggleAudio(path) { - if (path === "") return - var same = audioPlayer.running && root.playingAudio === path - if (audioPlayer.running) audioPlayer.running = false - if (same) { - root.playingAudio = "" - root.note = "stopped" - noteTimer.restart() + property bool audioPlaying: false + property int audioResetToken: 0 + property string pendingAudioId: "" + property string pendingAudioChat: "" + Loader { + id: audioPlayback + active: root.playingAudio !== "" + source: "AudioPlayback.qml" + onLoaded: { + item.source = Qt.binding(function() { return ((root.attMetrics[root.playingAudio] || {}).audio || {}).playbackUrl || "" }) + item.playing = Qt.binding(function() { return root.audioPlaying }) + item.muted = Qt.binding(function() { return root.audioMuted }) + item.resetToken = Qt.binding(function() { return root.audioResetToken }) + } + } + Connections { + target: audioPlayback.item + function onPlaybackEnded() { root.audioPlaying = false } + function onPlaybackFailed() { root.failAudio(root.playingAudio) } + } + function toggleAudio(att) { + var id = String(att.id || "") + if (id === "" || !root.inThread || !root.surfaceOpen) return + if (root.pendingAudioId === id) { root.stopAudio(); return } + if (root.playingAudio === id && root.attFiles[id]) { + root.audioPlaying = !root.audioPlaying return } - root.playingAudio = path - audioPlayer.command = ["sh", "-c", - 'if command -v mpv >/dev/null 2>&1; then exec mpv --no-video --force-window=no --no-terminal --really-quiet -- "$1"; else exec xdg-open "$1"; fi', - "sh", path] - audioPlayer.running = true - root.note = "playing, click again to stop" - noteTimer.restart() + root.stopAudio() + root.pendingAudioId = id + root.pendingAudioChat = String(root.active.chat) + // Fetch again before a fresh play: the cache may have been evicted. + // An in-flight auto-fetch also satisfies this captured play intent. + root.enqueueFetch(att, "audio-play", false) + } + function settleAudioFetch(id, ok, error) { + if (root.pendingAudioId !== id) return + var belongsHere = root.surfaceOpen && root.inThread + && String(root.active.chat) === root.pendingAudioChat + root.pendingAudioId = "" + root.pendingAudioChat = "" + if (ok && belongsHere && root.attFiles[id]) { + var metadata = (root.attMetrics[id] || {}).audio + if (metadata && metadata.playable === true) { + root.playingAudio = id + root.audioPlaying = true + } else if (metadata && metadata.externalDemuxer) { + Quickshell.execDetached(["mpv", "--no-video", "--demuxer=lavf", "--demuxer-lavf-format=" + metadata.externalDemuxer, "--", String(root.attFiles[id])]) + root.note = "Opened audio externally" + } else root.note = "Audio preview unavailable β€” right-click to save the file" + } else if (!ok && belongsHere) { + // Play can attach to an automatic fetch whose action remains empty. + // Its captured intent still deserves the failure reason before clearing. + root.note = "fetch failed β€” " + String(error || "could not load audio message").replace(/^error:\s*/, "") + } } - Process { - id: audioPlayer - onExited: function(code, status) { root.playingAudio = "" } + function failAudio(id) { + var requested = root.playingAudio === id || root.pendingAudioId === id + if (requested) root.stopAudio() + // Clear the source so retry reloads even when the cache URL is unchanged. + var files = Object.assign({}, root.attFiles) + files[id] = "" + root.attFiles = files + if (requested) root.note = "Could not play this audio message β€” try again or save the file" } function stopAudio() { - if (audioPlayer.running) audioPlayer.running = false + root.audioPlaying = false root.playingAudio = "" + root.pendingAudioId = "" + root.pendingAudioChat = "" + root.audioResetToken++ } Component.onDestruction: root.stopAudio() @@ -1123,7 +1167,7 @@ FocusScope { * make executable (scripts, .desktop, unknown blobs) is saved and named, * never launched (Codex audit #10). */ function openableMime(m) { - m = String(m || "") + m = String(m || "").toLowerCase() return m.indexOf("image/") === 0 || m.indexOf("video/") === 0 || m.indexOf("audio/") === 0 || m === "application/pdf" || m === "text/plain" || m === "text/vcard" || m === "text/calendar" } @@ -1205,6 +1249,7 @@ FocusScope { else root.openAttachment(att) } function openAttachment(att) { + if (String(att.mime || "").toLowerCase().indexOf("audio/") === 0 && audioPlayback.status !== Loader.Error) { root.toggleAudio(att); return } var id = String(att.id || "") if (id === "") return if (attFiles[id] === "") { // failed marker β€” clear it so a retry runs @@ -1995,9 +2040,10 @@ FocusScope { if (!isFinite(pixelHeight) || pixelHeight < 1 || pixelHeight > 100000) pixelHeight = 0 if (!keepInline) { var metrics = Object.assign({}, root.attMetrics) - metrics[id] = { pixelRatio: ratio, pixelWidth: pixelWidth, pixelHeight: pixelHeight, animated: d.animated === true, renderMime: String(d.renderMime || ""), animationUnavailable: d.animationUnavailable === true } + metrics[id] = { pixelRatio: ratio, pixelWidth: pixelWidth, pixelHeight: pixelHeight, animated: d.animated === true, renderMime: String(d.renderMime || ""), animationUnavailable: d.animationUnavailable === true, audio: d.audio || null } root.attMetrics = metrics } + root.settleAudioFetch(id, d.ok === true, d.error || "") if (d.ok === true && root.fetchJobAction === "copy") { // The Mac converts HEIC/HEIF to JPEG on the way (fetch.ts wantsJpeg), // so the clipboard type must say what the bytes are. Path and type @@ -2014,8 +2060,11 @@ FocusScope { saveProc.running = true } else root.note = "a save dialog is already open" } else if (d.ok === true && root.fetchJobAction === "open") { - if (String(root.fetchJobMime || "").indexOf("audio/") === 0) { - root.toggleAudio(String(d.path || "")) + // The ordinary Open route remains the fallback when QtMultimedia cannot load. + if (String(root.fetchJobMime || "").toLowerCase().indexOf("audio/") === 0) { + if (d.audio && d.audio.externalDemuxer) + Quickshell.execDetached(["mpv", "--no-video", "--demuxer=lavf", "--demuxer-lavf-format=" + d.audio.externalDemuxer, "--", String(d.path || "")]) + else root.note = "Audio format unavailable β€” right-click to save" } else if (root.openableMime(root.fetchJobMime)) { Quickshell.execDetached(["xdg-open", String(d.url || "")]) } else { @@ -2029,6 +2078,7 @@ FocusScope { else if (d.ok !== true && root.fetchJobAction !== "") root.note = "fetch failed β€” " + String(d.error || "unknown error").replace(/^error:\s*/, "") } catch (e) { + root.settleAudioFetch(id, false, "invalid fetch response") var m2 = Object.assign({}, root.attFiles) m2[id] = "" root.attFiles = m2 @@ -2036,6 +2086,7 @@ FocusScope { } } onExited: function(code, status) { + root.settleAudioFetch(root.fetchingId, false, "fetch process ended") root.fetchingId = "" Qt.callLater(root.pumpFetch) } @@ -2398,7 +2449,7 @@ FocusScope { || text === "u" || text === "U" || (text >= "1" && text <= "9") if (!jump) return false - if (searchField.activeFocus || newField.activeFocus || bubbleFocused) return false + if (searchField.activeFocus || newField.activeFocus || bubbleFocused || focusedAudio !== null) return false if (composeField.activeFocus && (composeField.text.length > 0 || root.attachCount > 0)) return false return handleTextKey(text) === true @@ -3564,6 +3615,7 @@ FocusScope { readonly property bool failed: chipRow.fileUrl === "" readonly property var imageMetrics: root.attMetrics[chipRow.attId] || ({}) readonly property bool isSticker: !!modelData.sticker + readonly property bool isAudio: String(modelData.mime || "").toLowerCase().indexOf("audio/") === 0 readonly property bool showImage: !isSticker && root.isImageMime(chipRow.modelData.mime) && chipRow.fileUrl !== undefined && chipRow.fileUrl !== "" @@ -3706,9 +3758,47 @@ FocusScope { } TapbackPill { visible: chipRow.pillHere; mine: bubbleRow.mine; tapbacks: bubbleRow.shownTapbacks } } + Loader { + id: audioCardLoader + active: chipRow.isAudio && audioPlayback.status !== Loader.Error + visible: active + Layout.preferredWidth: Math.min(Style.space(270), content.width * 0.80) + Layout.preferredHeight: Style.space(64) + sourceComponent: Component { AudioMessage { + anchors.fill: parent + objectName: chipRow.isAudio ? "blipAudio:" + chipRow.attId : "" + Layout.preferredWidth: Math.min(Style.space(270), content.width * 0.80) + Layout.preferredHeight: Style.space(64) + // Only recognized audio containers reach the media + // decoder; an audio MIME alone could conceal a playlist. + playback: root.playingAudio === chipRow.attId ? audioPlayback.item : null + waveform: (chipRow.imageMetrics.audio || {}).waveform || [] + durationMs: Number((chipRow.imageMetrics.audio || {}).durationMs || 0) + playing: root.playingAudio === chipRow.attId && root.audioPlaying + muted: root.audioMuted + resetToken: root.audioResetToken + loading: root.fetchingId === chipRow.attId || root.fetchQueue.some(function(job) { return job.id === chipRow.attId }) + failed: chipRow.attId === "" || chipRow.failed || (!!chipRow.fileUrl && (chipRow.imageMetrics.audio || {}).playable !== true) + fillColor: bubbleRow.mine ? root.mineFill : root.theirsFill + textColor: bubbleRow.mine ? root.mineText : root.theirsText + fontFamily: root.fontFamily + fontSize: root.fontCaption + id: audioMessage + onControlsFocusedChanged: { + if (controlsFocused) root.focusedAudio = audioMessage + else if (root.focusedAudio === audioMessage) root.focusedAudio = null + } + Component.onDestruction: { + if (root.focusedAudio === audioMessage) root.focusedAudio = null + } + onEscapeRequested: { root.focusedAudio = null; composeField.forceActiveFocus() } + onToggleRequested: root.toggleAudio(chipRow.modelData) + onContextRequested: root.openAttachmentMenu(bubbleRow.modelData, chipRow.modelData) + TapbackPill { visible: chipRow.pillHere; mine: bubbleRow.mine; tapbacks: bubbleRow.shownTapbacks } + } } } Rectangle { TapbackPill { visible: chipRow.pillHere; mine: bubbleRow.mine; tapbacks: bubbleRow.modelData.tapbacks } - visible: !chipRow.showImage && !chipRow.isSticker + visible: !chipRow.showImage && !chipRow.isSticker && (!chipRow.isAudio || audioPlayback.status === Loader.Error) Layout.preferredWidth: Math.ceil(chipText.implicitWidth) + Style.space(18) Layout.preferredHeight: Math.ceil(chipText.implicitHeight) + Style.space(12) radius: Style.space(14) diff --git a/audio-metadata.ts b/audio-metadata.ts new file mode 100644 index 00000000..1fec0409 --- /dev/null +++ b/audio-metadata.ts @@ -0,0 +1,101 @@ +/** Audio-only metadata/envelope extraction. All PCM remains in memory. */ +import { spawnSync } from "node:child_process"; +import { closeSync, constants, fstatSync, openSync, readSync, utimesSync, writeFileSync, renameSync, unlinkSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { AUDIO_ANALYSIS_SECONDS, AUDIO_SAMPLE_RATE, waveformFromPcm16 } from "./audio-waveform"; + +export interface AudioMetadata { durationMs: number; waveform: number[]; unavailable: boolean; playable: boolean; externalDemuxer?: string; playbackUrl?: string } +const MAX_INPUT = 100 * 1024 * 1024; + +/** Explicit demuxers prevent disguised playlists from loading referenced + * resources. The cached CAF may already be Ogg after the existing rewrap. */ +export function audioDemuxer(header: Uint8Array): string { + const text = (a: number, b: number) => String.fromCharCode(...header.subarray(a, b)); + if (header.length < 12) return ""; + if (text(0, 4) === "RIFF" && text(8, 12) === "WAVE") return "wav"; + if (text(0, 4) === "OggS") return "ogg"; + if (text(0, 4) === "caff") return "caf"; + if (text(0, 6) === "#!AMR\n") return "amr"; + if (text(0, 4) === "FORM" && ["AIFF", "AIFC"].includes(text(8, 12))) return "aiff"; + if (header[0] === 0x1a && header[1] === 0x45 && header[2] === 0xdf && header[3] === 0xa3) return "matroska"; + if (text(0, 4) === "fLaC") return "flac"; + if (text(4, 8) === "ftyp") return "mov"; + if (text(0, 3) === "ID3" || (header[0] === 255 && (header[1]! & 0xe0) === 0xe0 && (header[1]! & 6) !== 0)) return "mp3"; + if (header[0] === 255 && (header[1]! & 0xf6) === 0xf0) return "aac"; + return ""; +} + +export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { + const empty: AudioMetadata = { durationMs: 0, waveform: [], unavailable: true, playable: false }; + let fallback = empty; + let fd: number | undefined; + try { + fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); + const st = fstatSync(fd); + if (!st.isFile() || st.uid !== process.getuid() || st.size <= 0 || st.size > MAX_INPUT) return empty; + // Generated PCM WAV is the only file ever handed to Qt. The source + // fingerprint invalidates both waveform and normalized audio together. + const hash = createHash("sha256"), chunk = Buffer.alloc(65536); + for (let offset = 0; offset < st.size;) { + const n = readSync(fd, chunk, 0, Math.min(chunk.length, st.size - offset), offset); + if (!n) return empty; + hash.update(chunk.subarray(0, n)); offset += n; + } + const playbackPath = path + `.pcm-${hash.digest("hex").slice(0, 24)}.wav`; + try { + const cached = openSync(playbackPath, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const cs = fstatSync(cached); + if (cs.isFile() && cs.uid === process.getuid() && cs.size > 44 && cs.size <= AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + 44) { + const data = Buffer.alloc(cs.size); + if (readSync(cached, data, 0, data.length, 0) === data.length && data.toString("ascii", 0, 4) === "RIFF" && data.toString("ascii", 8, 16) === "WAVEfmt ") { + const now = new Date(); utimesSync(playbackPath, now, now); + return {durationMs: Math.round((data.length - 44) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(44)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; + } + } + } finally { closeSync(cached); } + } catch { /* no normalized cache entry */ } + const header = Buffer.alloc(16); const length = readSync(fd, header, 0, header.length, 0); + const format = audioDemuxer(header.subarray(0, length)); + if (!format) return empty; + empty.externalDemuxer = format; + const input = ["-protocol_whitelist", "file,pipe", "-f", format, "-i", path]; + const probe = runner("ffprobe", ["-v", "error", ...input, "-select_streams", "a:0", + "-show_entries", "format=duration:stream=codec_type", "-of", "json"], + { timeout: 10000, maxBuffer: 16384 }); + if (probe.status !== 0) return empty; + const data = JSON.parse(String(probe.stdout)); + if (!Array.isArray(data.streams) || !data.streams.some((s: any) => s.codec_type === "audio")) return empty; + fallback = { ...empty, playable: false }; + const seconds = Number(data.format?.duration); + if (!Number.isFinite(seconds) || seconds <= 0 || seconds > 86400) return { ...empty, playable: false }; + const durationMs = Math.round(seconds * 1000); + fallback = { durationMs, waveform: [], unavailable: true, playable: false }; + // A waveform must describe the complete recording. Don't display a + // truncated ten-minute envelope against a longer recording's timeline. + if (seconds > AUDIO_ANALYSIS_SECONDS) return { durationMs, waveform: [], unavailable: true, playable: false }; + const result = runner("ffmpeg", ["-nostdin", "-v", "error", "-threads", "1", ...input, + "-map", "0:a:0", "-vn", "-sn", "-dn", "-t", String(AUDIO_ANALYSIS_SECONDS), + "-ac", "1", "-ar", String(AUDIO_SAMPLE_RATE), "-c:a", "pcm_s16le", "-f", "s16le", "pipe:1"], + { timeout: 15000, maxBuffer: AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + 16384 }); + if (result.status !== 0 || !Buffer.isBuffer(result.stdout)) return { durationMs, waveform: [], unavailable: true, playable: false }; + const decodedSeconds = result.stdout.length / (2 * AUDIO_SAMPLE_RATE); + if (Math.abs(decodedSeconds - seconds) > Math.max(0.25, seconds * 0.02)) + return { durationMs, waveform: [], unavailable: true, playable: false }; + const waveform = waveformFromPcm16(result.stdout); + if (!waveform.length) return empty; + const wav = Buffer.alloc(44); + wav.write("RIFF", 0); wav.writeUInt32LE(result.stdout.length + 36, 4); wav.write("WAVEfmt ", 8); + wav.writeUInt32LE(16, 16); wav.writeUInt16LE(1, 20); wav.writeUInt16LE(1, 22); + wav.writeUInt32LE(AUDIO_SAMPLE_RATE, 24); wav.writeUInt32LE(AUDIO_SAMPLE_RATE * 2, 28); + wav.writeUInt16LE(2, 32); wav.writeUInt16LE(16, 34); wav.write("data", 36); wav.writeUInt32LE(result.stdout.length, 40); + const tmp = playbackPath + `.tmp-${process.pid}`; + try { + writeFileSync(tmp, Buffer.concat([wav, result.stdout]), {mode:0o600, flag:"wx"}); + renameSync(tmp, playbackPath); + } finally { try { unlinkSync(tmp); } catch {} } + return { durationMs, waveform, unavailable: false, playable: true, playbackUrl:pathToFileURL(playbackPath).href }; + } catch { return fallback; } + finally { if (fd !== undefined) closeSync(fd); } +} diff --git a/audio-waveform.test.ts b/audio-waveform.test.ts new file mode 100644 index 00000000..513717d8 --- /dev/null +++ b/audio-waveform.test.ts @@ -0,0 +1,113 @@ +import { describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { AUDIO_ANALYSIS_SECONDS, AUDIO_SAMPLE_RATE, durationLabel, fitWaveform, normalizeWaveform, + playbackFraction, seekPosition, waveformFromPcm16 } from "./audio-waveform"; +import { analyzeAudio, audioDemuxer } from "./audio-metadata"; + +function pcm(samples: number[]): Buffer { + const bytes = Buffer.alloc(samples.length * 2); + samples.forEach((sample, i) => bytes.writeInt16LE(sample, i * 2)); + return bytes; +} +function wavHeader(): Buffer { + const h = Buffer.alloc(44); h.write("RIFF"); h.write("WAVE", 8); return h; +} + +describe("measured audio presentation", () => { + test("silence stays flat; changes in the source produce corresponding envelope", () => { + expect(waveformFromPcm16(pcm([0,0,0,0]), 2)).toEqual([0,0]); + expect(waveformFromPcm16(pcm([100,-100,1600,-1600,0,0,400,-400]), 4)).toEqual([0.25,1,0,0.5]); + expect(waveformFromPcm16(pcm([-32768,32767]), 2)[0]).toBe(1); + expect(waveformFromPcm16(new Uint8Array())).toEqual([]); + expect(waveformFromPcm16(pcm([100]), 200)).toEqual([]); + expect(waveformFromPcm16(new Uint8Array(AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + 1))).toEqual([]); + }); + test("malformed or oversize UI data is bounded, and narrow waveforms retain peaks", () => { + expect(normalizeWaveform([NaN,Infinity,-1,0.5,2,"1"])).toEqual([0,0,0,0.5,1,0]); + expect(normalizeWaveform(Array(97).fill(1))).toEqual([]); + expect(normalizeWaveform({length:AUDIO_SAMPLE_RATE * 2.50})).toEqual([]); + expect(fitWaveform([0,0.8,0.2,1,0.4,0.7], 12)).toEqual([0.8,1,0.7]); + expect(fitWaveform([1], 1)).toEqual([1]); + expect(fitWaveform([1], NaN)).toEqual([]); + }); + test("duration and seeking clamp invalid positions", () => { + expect(durationLabel(0)).toBe("0:00"); expect(durationLabel(65300)).toBe("1:05"); + expect(durationLabel(NaN)).toBe("0:00"); expect(durationLabel(-1)).toBe("0:00"); + expect(playbackFraction(20,10)).toBe(1); expect(playbackFraction(-1,10)).toBe(0); + expect(playbackFraction(1,0)).toBe(0); expect(playbackFraction(NaN,1)).toBe(0); + expect(seekPosition(0.25,1000)).toBe(250); expect(seekPosition(5,1000)).toBe(1000); + }); +}); + +describe("bounded private audio analysis", () => { + test("demuxer selection rejects disguised playlists and checks actual bytes", () => { + expect(audioDemuxer(wavHeader())).toBe("wav"); + expect(audioDemuxer(Buffer.from("#EXTM3U\nfile:///private"))).toBe(""); + expect(audioDemuxer(Buffer.from("external resource"))).toBe(""); + for (const [magic, format] of [["OggS","ogg"],["caff","caf"],["fLaC","flac"],["ID3","mp3"]]) { + const h = Buffer.alloc(16); h.write(magic!); expect(audioDemuxer(h)).toBe(format); + } + }); + test("the real duration and complete PCM envelope survive, with fixed caps and explicit demuxer", () => { + const dir = mkdtempSync(join(tmpdir(), "blip-audio-test-")); + const path = join(dir,"sample.wav"); writeFileSync(path,wavHeader(),{mode:0o600}); + try { + const calls: string[] = []; + const runner = ((cmd: string, args: string[], opts: any) => { + calls.push(cmd); expect(args).toContain("file,pipe"); expect(args).toContain("wav"); + expect(args).not.toContain("http"); expect(opts.timeout).toBeLessThanOrEqual(15000); + if (cmd === "ffprobe") return { status:0, stdout: JSON.stringify({streams:[{codec_type:"audio"}],format:{duration:"2.5"}}) }; + expect(args).toContain("-nostdin"); + expect(args[args.indexOf("-t") + 1]).toBe(String(AUDIO_ANALYSIS_SECONDS)); + expect(opts.maxBuffer).toBeLessThanOrEqual(AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + 16384); + return { status:0, stdout:pcm(Array.from({length:AUDIO_SAMPLE_RATE * 2.5}, (_,i) => i < 1000 ? 0 : 300)) }; + }) as never; + const result = analyzeAudio(path,runner); + expect(result.durationMs).toBe(2500); expect(result.unavailable).toBe(false); + expect(result.playable).toBe(true); + expect(result.waveform.length).toBe(64); expect(result.waveform[0]).toBe(0); + expect(calls).toEqual(["ffprobe","ffmpeg"]); + expect(result.playbackUrl).not.toBe("file://" + path); + const cached = analyzeAudio(path, (() => { throw Error("cache must avoid analysis"); }) as never); + expect(cached.playable).toBe(true); expect(cached.waveform).toEqual(result.waveform); + const link = join(dir,"link.wav"); symlinkSync(path,link); + expect(analyzeAudio(link,runner).unavailable).toBe(true); expect(calls.length).toBe(2); + } finally { rmSync(dir,{recursive:true,force:true}); } + }); + test("long recordings retain duration without a misleading truncated waveform; failures remain playable", () => { + const dir = mkdtempSync(join(tmpdir(), "blip-audio-test-")); + const path = join(dir,"sample.wav"); writeFileSync(path,wavHeader()); + try { + for (const seconds of [601, NaN, -1]) { + let calls = 0; + const r = analyzeAudio(path, (() => { calls++; return {status:0,stdout:JSON.stringify({streams:[{codec_type:"audio"}],format:{duration:seconds}})}; }) as never); + expect(calls).toBe(1); expect(r.waveform).toEqual([]); expect(r.unavailable).toBe(true); + expect(r.durationMs).toBe(seconds===601 ? 601000 : 0); + } + const timedOut = analyzeAudio(path, ((cmd: string) => cmd === "ffprobe" + ? {status:0,stdout:'{"streams":[{"codec_type":"audio"}],"format":{"duration":"2"}}'} : {status:null,stdout:Buffer.alloc(0),error:Error("ETIMEDOUT")}) as never); + expect(timedOut.durationMs).toBe(2000); expect(timedOut.unavailable).toBe(true); + expect(timedOut.playable).toBe(false); + const missing = analyzeAudio(path, ((cmd: string) => { + if (cmd === "ffprobe") return {status:0,stdout:'{"streams":[{"codec_type":"audio"}],"format":{"duration":"2"}}'}; + throw Error("ENOENT"); + }) as never); + expect(missing.durationMs).toBe(2000); expect(missing.playable).toBe(false); + const noAudio = analyzeAudio(path, (() => ({status:0,stdout:'{"streams":[{"codec_type":"video"}],"format":{"duration":"2"}}'})) as never); + expect(noAudio.playable).toBe(false); + } finally { rmSync(dir,{recursive:true,force:true}); } + }); +}); + +test("a lying duration never produces playable normalized audio", () => { + const dir=mkdtempSync(join(tmpdir(),"blip-audio-duration-")); const path=join(dir,"voice.wav"); + writeFileSync(path,wavHeader()); + try { + const r=analyzeAudio(path,((cmd:string)=>cmd === "ffprobe" + ? {status:0, stdout:JSON.stringify({streams:[{codec_type:"audio"}],format:{duration:"10"}})} + : {status:0,stdout:pcm(Array(AUDIO_SAMPLE_RATE).fill(100))}) as never); + expect(r.playable).toBe(false); expect(r.playbackUrl).toBeUndefined(); + } finally {rmSync(dir,{recursive:true,force:true});} +}); diff --git a/audio-waveform.ts b/audio-waveform.ts new file mode 100644 index 00000000..e272a062 --- /dev/null +++ b/audio-waveform.ts @@ -0,0 +1,64 @@ +/** Pure presentation math shared by Bun and QML. Samples represent decoded + * audio; unavailable audio uses an empty waveform, never decorative data. */ +export const AUDIO_WAVEFORM_BARS = 64; +export const AUDIO_SAMPLE_RATE = 24000; +export const AUDIO_ANALYSIS_SECONDS = 600; + +export function normalizeWaveform(value: unknown): number[] { + if (!Array.isArray(value) || value.length > 96) return []; + return value.map(v => typeof v === "number" && Number.isFinite(v) ? Math.max(0, Math.min(1, v)) : 0); +} + +export function fitWaveform(value: unknown, width: number): number[] { + const input = normalizeWaveform(value); + if (!input.length || !Number.isFinite(width) || width <= 0) return []; + const count = Math.min(input.length, Math.max(1, Math.floor(width / 4))); + const output: number[] = []; + for (let i = 0; i < count; i++) { + let peak = 0; + for (let j = Math.floor(i * input.length / count); j < Math.floor((i + 1) * input.length / count); j++) + peak = Math.max(peak, input[j]!); + output.push(peak); + } + return output; +} + +/** Bounded little-endian signed 16-bit PCM, RMS per equal-time bucket. + * Normalize within the recording, then compress amplitude for small bars. */ +export function waveformFromPcm16(bytes: Uint8Array, count = AUDIO_WAVEFORM_BARS): number[] { + const samples = Math.floor(bytes.length / 2); + if (!samples || bytes.length > AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + || !Number.isInteger(count) || count < 1 || count > 96) return []; + count = Math.min(count, samples); + const bins: number[] = []; + let largest = 0; + for (let bin = 0; bin < count; bin++) { + const first = Math.floor(bin * samples / count), last = Math.floor((bin + 1) * samples / count); + let sum = 0; + for (let i = first; i < last; i++) { + const unsigned = bytes[2 * i]! | (bytes[2 * i + 1]! << 8); + const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned; + sum += sample * sample; + } + const rms = Math.sqrt(sum / (last - first)); + bins.push(rms); largest = Math.max(largest, rms); + } + return largest === 0 ? bins : bins.map(v => Math.round(Math.sqrt(v / largest) * 1000) / 1000); +} + +export function durationLabel(milliseconds: number): string { + if (!Number.isFinite(milliseconds) || milliseconds < 0) return "0:00"; + const seconds = Math.floor(Math.min(milliseconds, 86400000) / 1000); + const minutes = Math.floor(seconds / 60), remainder = seconds % 60; + return String(minutes) + ":" + (remainder < 10 ? "0" : "") + String(remainder); +} + +export function playbackFraction(positionMs: number, durationMs: number): number { + if (!Number.isFinite(positionMs) || !Number.isFinite(durationMs) || durationMs <= 0) return 0; + return Math.max(0, Math.min(1, positionMs / durationMs)); +} + +export function seekPosition(fraction: number, durationMs: number): number { + if (!Number.isFinite(fraction) || !Number.isFinite(durationMs) || durationMs <= 0) return 0; + return Math.round(Math.max(0, Math.min(1, fraction)) * durationMs); +} diff --git a/docs/AUDIO.md b/docs/AUDIO.md new file mode 100644 index 00000000..6884233b --- /dev/null +++ b/docs/AUDIO.md @@ -0,0 +1,41 @@ +# Voice messages + +Audio attachments render as an inline player with a measured waveform, +remaining time, playback progress, and pause/resume. Clicking the waveform +seeks. Space or Return toggles the focused player; Left/Right seek five +seconds when the timeline has focus. Switching recordings, leaving the +conversation, or closing the panel stops playback. Loading an attachment +never starts it without an explicit play request. + +The player uses Qt Multimedia (`qt6-multimedia` on Arch; +`qml6-module-qtmultimedia` on Ubuntu). FFmpeg/FFprobe provide the existing CAF +to Opus conversion and the waveform analysis. Missing analysis tools or an +unsupported format produce an unavailable inline preview; the attachment's +Save action remains available. + +The waveform is computed from the recording, not generated decoration. The +fetcher verifies an audio container signature, selects its demuxer explicitly, +and confirms an audio stream with FFprobe before exposing a playable source. +It decodes mono 24 kHz signed PCM into memory and takes RMS values in 64 +equal-time buckets, normalized within that recording. Silence remains flat. +A narrow player groups those buckets without dropping peaks. + +Analysis has a 100 MB source ceiling, a ten-second metadata deadline, a +15-second decode deadline, and a 600-second/28.8 MB PCM limit. Longer recordings +retain their duration and a simple seek timeline instead of a misleading +partial waveform. Decoding failures likewise retain playable media when the +container and audio stream were verified. The verified mono PCM is wrapped in a generated WAV for Qt playback, cached privately and counted in LRU eviction; +only the existing private media cache contains the attachment itself. + +Tests use generated WAV tones and play muted. The Qt runtime checks actual +pause/resume, seek, completion, replay, and focus behavior, while TypeScript +tests cover envelope math, malformed metadata, and decoder bounds: + +```sh +bun test audio-waveform.test.ts +python3 scripts/test-audio-ui.py +``` + +![Synthetic inline players](img/inline-audio.png) + +Analysis happens only on a play click, never on thread open. Qt opens only the generated 24 kHz mono PCM WAV, never the received container. This bounds playback to ten minutes; unavailable normalization and missing tools fall back to external mpv or Save. The normalized file also caches the waveform by source content hash. The view owns the optional multimedia Loader independently of message delegates; model reloads preserve playback, leaving the conversation or closing the view stops it. Missing QtMultimedia falls back to an ordinary attachment chip. diff --git a/docs/img/inline-audio.png b/docs/img/inline-audio.png new file mode 100644 index 0000000000000000000000000000000000000000..71029d21856d0094cfce9ee2fbb6db471aecb574 GIT binary patch literal 5162 zcmb_=cQhPM^zW+CR*x=`)mChjL}V>NL|Z`+B6=??(TU!IjV`)it+oU~^j?-AEWs|3 zVu=W=gb+1^{q6U?f8L+(ocGS0nYlA(X70J?-nsLcPoj~b4v<{9;kue8005OO zMebvurnG-Tcf%+hqmQnIKLEh`{=Z6jX>2I%!@0H4(n0DvGE2(YKYPywEmC;*~?+4O*S zG5`M;X{UGw7n3xNIkk+xhumX_rmaBJJoNzAnjyF*$dDWhVK&fa9#@HIG)%HVyb_bT zV(zHH(h3l;KIF6qdhOD9qQP2+MZ`z*GZ zqjwEGFMU}85qEx}FTt5q7AM4QFTv^5XnNi3dXC@U@ROM#X`srKc$PsX7|Fv?!U|3T zF$}tUDg2~Lc;yNXE`D&k%7_=B!mcGhSM=krkNtDmI%Z0d+Il8FGLi?3kOs9 zv=O=zobIxK6R{*m4TCrynO(fdh$CMR?VH44Yer8G*gWqyFAACVq)fq8P#|uFq5M$0a8~XbqntsRX&P5@)t|Yl;&P=^}@{fP` ziGF4U`{Vm`XML%oTbK+6b2g8YqkK^9x+MQOi-mYKczi;JV$pEi>ltTzuV1(_ zq<6YN!R}yc=fLG~ zx|3=d$H71O4-;R{<)yhIABhQ}#9}`hacUV5JOXc9QRULdG~COIBZX)gJ6rq+i6MpD z`=0ngPM@G)K$vdvt2Vu9K!_@d)}OhZ`ccPPz4u&T5baA{%oa}N@Ct(+{iBn%VrrH; zdifjj5#)KxI#^F7M{gsv^9kfK6CKX6A15qA8-7DIsEWrPv8ys&QsZmRT&qP17pV;0 zpMK@>8Z%Z#(fpe1)K`a-A0yalOIurqzp*EaFjSi+*VO2wIkODy`p`wcc9O`WZx@4m zczW)AN#xY3?(N8UpILEA6W34S+HL_^Il9^O@JvffOOLPGKPa7;)NZF;bm(xOg6SwP^}4ce zUEK|+81;bQ5e7>>8FUWrvkU+X+#VTvRU^O_FjfWDs}TrNEf2xcJt$L&q-Ph%t)ov{ z$;6@&Ir@r9XzkRK6Ml8~C%N=yW~dapg~0h^n6k2p4hQW$Gqaa!MPtsDC!xRksb{5g zg_&ONCnxiwBS!eqcKwk<`&0evo;Mz;)c-`Ge%ftIq-;a<*P$g>semsmYAN0!m9sQK z?83I^*zN|9U(liV*&lFY*gqA~%yPMU^Rc;&JrA0)>GT`E(;xEGzj=#;NyvuzJ5w`W z1b0PJLZ-|8i)i}4(P3V@b&qr;$p?^{=lpNq*a zPW%Esy4x@&Sxl1c{)MYGR}22Al&ryUX&SnVqyh*|=*2qBeplBSl2i0r2*BNF@827# zXCDla)4Hk*>;1BR+`i|=a<#dUp6u4Uvqb*1IhQW1@>$Rj>Rq|+q*HZA@ccsYB4yd> zi9io}MiEvJ_HR$pF3hub-{0{(j>cVm0>)jiwin$tS=hc1zVBIh$J;kBvW4WcoDgZn zZE#wU&k;3p-L(#BQk|`bvA&CGKQ5oEjq3DXvRWJr3H~&D^R<3p>=E~pGPjXcK+@X< z34ZIWD^Vi_Vix`jiTi7p0yd7?+7OW?y^~{#NxZhf+Ju%W^x`{2*66lbm#tyd%c_gl ztxxCKXXg7OV_$C3#L3=OnSAh7GFMqPc{no*|IVLOKYHbCZhHEfT1@3RKQ~=;YIS_K+N_Ff2~c>K=-*Yi_`AEX&2BbHRPu*ss%gmJZO)hXz`Jxy{UgJHfg( zvSHnGL<_eYJ$oZtR#sL^k!L70`5#taVrzE>{qJY7zQler`n-E-mMY(|-{|y1hQ4|- zt3Cnh_a0|z1j^`GN&9v+B49GdJaT&=hRwTc11+o?x*U(|>VloG0okd3J&1KMT^hLU zy>C_Q9r=aezt;V`?|7{+N`b}c{1lkXgSwIQLp<-_VbW!_qF=@!4==CNO>^0$cMeXe zQEUJnkna@EuhSZJwVs&j@m7E2%u|*$@tQb4Z+G5j1mbg%BfGl}l3cl287?8uETr;qNo6cR;Pw*v z$T__@@)UtW2~&K%m!+g0povjnH}15i_#%)vpR~5QDS>VulgVk(1nrIW*VMqC%da-3 zZ+1lv0p}ZR&mXAWqXLN!u*SPs)TeoifrjMp8xCfvl34~`^U8ckc^K?RRpV}AUteEw z@vYjWse-xdNk2Z*nUAU#=u5aw?O1mHu?2As=ZnFm&FfDtj(;Da83)9u(doRFmfvW) z0<4V}a*@|<8n~J02nr#^{w5*;y+u=`<}YvS%~j%*AHz_6n>EsE|8|E0LQe{w)(Z#; zE%cqUp-?Cy5e=_;nI3UKxvr6GuT!V)wr_Nj6oD)(yDCvrch#!l5`Z?AAtpzw=_gce z0+dV{4k~xC_#3W@p4wMw04G5nsL1ZB*xenS{6F$}&0hP~NEyYxJc!cooo@2vk#V$Kyw8PhG&3e^tOA6ivJooDKolqz;OGQE0}i@2^ReAf zby4X{*K2_4Mpv2PyGwF|dOiR!fwLvC?6H@S&a zP)po<1-KRO!uR;FD5j}?D^%&<&QK#?`=$W_PkBZa+7N&NS`L^XC!p;A8ITP~J*&6y z)7!GG6z+eoH=#HuBf992==%wdO&Q(LDh*DMW%2t<)#=^6(49kQg6u2#E`}qQ-8j<- zU3t7b*5yGrK(_WM;H)x%{ZdeM*!)8VzZq6W)X`@Uq%GWuZn+0o`vts9e4@+oD z=j`wQ-jsrLAOm>-M@-Cpl|hHJ!gQErKA}9TaLFw(KxL-Yfh_l~@U=z(KTus*i11sx zlW+ZH&vWo}r-$8HkyBz+68C5fqTxopwK`k2GFQXvj`A*;vyq9L<Lp~01j!nCLP49i|W;3sq-}R0N9=@l1 zI)&ZbXQJS@L0a4|KHB$`Oxg%6tj>7w=_p4sZgFc|uI7lDo}KizQH@Oevsfu04&~4r zf(vPh+9o_nR*rqx?YpKP7>^+$F#*rp_DW0h(8 z!^D$l;cMCScG-NbwK_UnDp_;|sEU9Xy&M6|Wu7VjKdnr#@=F||tXX-B;+mWKCw0tb z)H*0ey@cBxruq4@wD~i!2^=r+-!nMN6>~f~l2D=^kKp%iv6fL`u*sVz8k0foKAPLt zhj+=eAb{+B*?;M>`r6+&`!P(nsJ}MUi;Ji4bFhw4i;L^rDgZ(i=}Yg`VujPd7}_i1 z80co5!uQX7ar780s^XWs4PnTxY^9(_C($=MyNcWxfW458j#@YBSn-MU{h|W%mZ@8W zKRph%#$~f30=gazj~)qA!t1L>EtW3EUw*vg{rWe%l2r3)i_wroaprre0o2eK{CACo z^UR7Yx0x>7#wf2z?HLCX+cLm|4uhqe+RBhQ6FUDrcaTF!2qLf#{E@V0pv+@O?W8f& zNvkb8elX096rbgMxg-9MM&K;$JiXA~@TDx~3uQ7hX}Z~bIxB2_8D%<6!^Ydf=!9No ziNDDo?m57*^dU-43tso!#HRg~ZC6t6!yrP>dLI27r~m!E!^Rf~t-I$R)DjPl$y!#d zaU5PAsb9RkCzih6NV(l%|Kz8?s_*e=z?5RB+h;^a$!*vEUiqk}?M4FDa{8bK!P+Kq z%huT9rH7R(cY0z(m!pLPz}CCr>fc5p4uT%ej6}$6zCin27|2WWTRY{$jb1X&S{O-A zd-#;T;p$X*eAGL1P+r$^=G3sx6G1lF&P zW3RAP^I!1}48k%VG}u)b<{Yt1qxwsi_h{_NKG%s|`L0qeY6d~xrEIqj#&0Vqha4Ce zmy<8Jh-b?EWA};bwba%>xk)^d zxwR)IM@!+?N?V>}#k%_1`f=yY)ZC2@iPMPRGg@CFL}%}4Xl7@xk53WW%3U#gum9nJ;vjGDcG1xqlczVAI{yrHK8{$uYi-1NA11E~@na_A0)qgd-n~q;& zTTjHPdj7KZg=zCsxQ8KVk!`?Pb`qRMNkGPAIAXul(YQ;UMk#%T*-03xv2U3kxs)LS zhklKHQ?O)6ibN*7c~UBQG!1WlV;~O7q}zObzairL9he-}oORHgZnB(Y+p5V0%-Dr3!w4CR>glmMOn z2_NU)@X1J6&wdWeK8BhzP3+O7%bH8du_&?H@l~vOc~7+_Xx|o?)BO&X$>bD`k&ICe zO)GTBpcIp84XxfIe{=|HCpbQ`B^$=oQj6|869kp*cW-27ZFG?~j(b~n$Q1jfr0`e> z@o1|&OkJ5P08Cx@Z$MC0jeB9ZvDc&M7vAhEJg?~^^#I3eFKj({Rv#DnS`QM3c5nabuFmw&y! z0tPC+#}d~ct2k7 z6{q{*imvBh9=CY;{xrOtlp*e8=pIuene8nXl?W*FG-x7pFG*bzGRjE-yF-ZzT)xQ4 z9x^97g/dev/null && return 0 + done + return 1 +} +if ! qt6_has qml/QtMultimedia/qmldir; then + echo "β€’ optional inline audio: sudo pacman -S --needed qt6-multimedia" +fi +if ! qt6_has 'plugins/multimedia/*mediaplugin.so'; then + echo "β€’ optional inline audio backend: sudo pacman -S --needed qt6-multimedia-ffmpeg" +fi if (( ${#missing[@]} )); then echo "βœ— missing on this machine: ${missing[*]}" >&2 echo " sudo pacman -S --needed ${pkgs[*]} (Omarchy/Arch names), then re-run blip-setup" >&2 exit 69 fi echo "βœ“ Linux prerequisites present (ssh scp bun jq notify-send wl-copy xdg-open)" -# Optional: voice messages. iPhone voice notes are CAF; ffmpeg turns them into -# Ogg and mpv plays them without a window. Blip works without either, a voice -# message just will not play. +# Optional analysis helpers: ffmpeg converts CAF and computes waveforms; +# ffprobe verifies the audio stream. Qt Multimedia plays inline. optional=() -for dep in ffmpeg mpv; do command -v "$dep" >/dev/null 2>&1 || optional+=("$dep"); done +optional_pkgs=() +for dep in ffmpeg ffprobe; do command -v "$dep" >/dev/null 2>&1 || optional+=("$dep"); done +if (( ${#optional[@]} )); then optional_pkgs+=(ffmpeg); fi +if ! command -v mpv >/dev/null 2>&1; then optional+=(mpv); optional_pkgs+=(mpv); fi if (( ${#optional[@]} )); then - echo "β€’ optional, for voice messages: sudo pacman -S --needed ${optional[*]}" + echo "β€’ optional voice-message tools missing: ${optional[*]}" + echo " sudo pacman -S --needed ${optional_pkgs[*]} (ffmpeg provides ffprobe)" fi # 0b. bun must be resolvable by the BAR, not just by this shell. Quickshell diff --git a/scripts/test-audio-session-ui.py b/scripts/test-audio-session-ui.py new file mode 100644 index 00000000..67668de0 --- /dev/null +++ b/scripts/test-audio-session-ui.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Exercise shipping audio play-intent handlers with a fake attachment fetcher.""" +from pathlib import Path +import os +import shutil +import subprocess +import tempfile + +repo = Path(__file__).resolve().parents[1] +source = (repo / 'BlipView.qml').read_text() + +def function(name): + start = source.index(' function ' + name + '(') + begin = source.index('{', start) + depth = 0 + for end in range(begin, len(source)): + if source[end] == '{': depth += 1 + elif source[end] == '}': + depth -= 1 + if depth == 0: return source[start:end + 1] + raise ValueError(name) + +qml = '''import QtQuick +import QtTest +Item { + id: root + property var audioPlayback: ({status: Loader.Ready}) + property int rowBudget: 1 + property int rowBatch: 1 + function retryBareAvatars() {} + @SURFACE@ + property bool surfaceOpen: true + property bool inThread: true + property var active: ({chat:"synthetic-a"}) + property var attFiles: ({}) + property var attMetrics: ({}) + property string playingAudio: "" + property bool audioPlaying: false + property int audioResetToken: 0 + property string pendingAudioId: "" + property string pendingAudioChat: "" + property string note: "" + property string fetchingId: "" + property string fetchJobAction: "" + property var requests: [] + function enqueueFetch(att, action, auto) { if (fetchingId === att.id) return; requests = requests.concat([{id:att.id,action:action,auto:auto}]) } + @FUNCTIONS@ + TestCase { + name: "AudioSession" + function init() { + root.audioPlayback = ({status: Loader.Ready}); root.stopAudio(); root.requests = []; root.note = ""; root.fetchingId = ""; root.fetchJobAction = ""; root.attFiles = ({}); root.attMetrics = ({}); + root.surfaceOpen = true; root.inThread = true; root.active = ({chat:"synthetic-a"}); + } + function complete(id, ok) { + if(ok) { var files=Object.assign({},root.attFiles); files[id]="file:///synthetic/voice.ogg"; root.attFiles=files; var metrics=Object.assign({},root.attMetrics); metrics[id]=({audio:{playable:true}}); root.attMetrics=metrics } + root.settleAudioFetch(id,ok) + } + function test_fetch_then_play_and_pause_resume_without_refetch() { + root.toggleAudio({id:"a"}); compare(root.requests.length,1); compare(root.requests[0].action,"audio-play"); + verify(!root.audioPlaying); complete("a",true); verify(root.audioPlaying); compare(root.playingAudio,"a"); + root.toggleAudio({id:"a"}); verify(!root.audioPlaying); + root.toggleAudio({id:"a"}); verify(root.audioPlaying); compare(root.requests.length,1); + } + function test_switching_chat_drops_late_completion() { + root.toggleAudio({id:"a"}); root.active=({chat:"synthetic-b"}); complete("a",true); + verify(!root.audioPlaying); compare(root.playingAudio,""); compare(root.pendingAudioId,""); + } + function test_closing_during_fetch_never_plays() { + root.toggleAudio({id:"a"}); root.surfaceOpen=false; complete("a",true); + verify(!root.audioPlaying); compare(root.pendingAudioChat,""); + } + function test_other_attachment_completion_does_not_consume_intent() { + root.toggleAudio({id:"a"}); complete("b",true); verify(!root.audioPlaying); compare(root.pendingAudioId,"a"); + complete("a",true); verify(root.audioPlaying); + } + function test_second_click_cancels_pending_play() { + root.toggleAudio({id:"a"}); root.toggleAudio({id:"a"}); complete("a",true); verify(!root.audioPlaying); + } + function test_switching_audio_stops_old_and_ignores_old_fetch() { + root.toggleAudio({id:"a"}); root.toggleAudio({id:"b"}); complete("a",true); verify(!root.audioPlaying); + complete("b",true); verify(root.audioPlaying); compare(root.playingAudio,"b"); + var token=root.audioResetToken; root.toggleAudio({id:"c"}); verify(!root.audioPlaying); verify(root.audioResetToken>token); + } + function test_failure_settles_intent_and_retry_fetches_again() { + root.toggleAudio({id:"a"}); complete("a",false); verify(!root.audioPlaying); compare(root.pendingAudioId,""); + root.toggleAudio({id:"a"}); compare(root.requests.length,2); + } + function test_play_during_auto_fetch_reports_its_failure_reason() { + root.fetchingId="a"; root.fetchJobAction=""; + root.toggleAudio({id:"a"}); compare(root.requests.length,0); compare(root.pendingAudioId,"a"); + root.settleAudioFetch("a",false,"error: attachment exceeds transfer limit"); + compare(root.note,"fetch failed β€” attachment exceeds transfer limit"); compare(root.pendingAudioId,""); verify(!root.audioPlaying); + // A conversation switch cancels the captured request and its error UI. + root.note=""; root.toggleAudio({id:"a"}); root.active=({chat:"synthetic-b"}); + root.settleAudioFetch("a",false,"old error"); compare(root.note,""); + } + function test_decoder_failure_clears_source_and_allows_same_url_retry() { + root.toggleAudio({id:"a"}); complete("a",true); root.failAudio("a"); + verify(!root.audioPlaying); compare(root.attFiles.a,""); + root.toggleAudio({id:"a"}); complete("a",true); verify(root.audioPlaying); + } + function test_unrecognized_audio_never_gets_play_intent() { + root.toggleAudio({id:"a"}); root.attFiles=({a:"file:///synthetic/disguised-playlist"}); + root.attMetrics=({a:{audio:{playable:false}}}); root.settleAudioFetch("a",true); + verify(!root.audioPlaying); compare(root.playingAudio,""); + } + function test_mixed_case_audio_and_missing_multimedia_fallback() { + root.openAttachment({id:"a",mime:"Audio/WAV"}); + compare(root.requests[0].action,"audio-play"); + root.requests=[]; root.audioPlayback=({status:Loader.Error}); + root.openAttachment({id:"b",mime:"Audio/WAV"}); + compare(root.requests[0].action,"open"); + } + function test_missing_id_or_closed_surface_cannot_request_play() { + root.toggleAudio({id:""}); root.surfaceOpen=false; root.toggleAudio({id:"a"}); compare(root.requests.length,0); + } + } +} +'''.replace('@SURFACE@', source[source.index(' onSurfaceOpenChanged: {'):source.index(' function pumpAvatar()', source.index(' onSurfaceOpenChanged: {'))]).replace('@FUNCTIONS@', '\n'.join(function(name) for name in ['toggleAudio','settleAudioFetch','failAudio','stopAudio','openAttachment'])) +runner = shutil.which('qmltestrunner') or '/usr/lib/qt6/bin/qmltestrunner' +with tempfile.TemporaryDirectory(prefix='blip-audio-session-') as tmp: + path=Path(tmp); (path/'tst_audio.qml').write_text(qml) + env={**os.environ,'QT_QPA_PLATFORM':'offscreen','QT_QUICK_BACKEND':'software','QT_QPA_PLATFORMTHEME':'generic'} + result=subprocess.run([runner,'-input',tmp],env=env) + raise SystemExit(result.returncode) diff --git a/scripts/test-audio-ui.py b/scripts/test-audio-ui.py new file mode 100644 index 00000000..f504e71e --- /dev/null +++ b/scripts/test-audio-ui.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Exercise shipping audio analysis/player using a muted synthetic WAV only.""" +import argparse +import json +import math +import os +from pathlib import Path +import shutil +import struct +import subprocess +import tempfile +import wave + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--capture", type=Path) + args = parser.parse_args() + runner = shutil.which("qmltestrunner") or next((str(p) for p in ( + Path("/usr/lib/qt6/bin/qmltestrunner"), Path("/usr/lib/qt6/libexec/qmltestrunner")) if p.is_file()), None) + if not runner: + raise SystemExit("Qt 6 qmltestrunner required") + with tempfile.TemporaryDirectory(prefix="blip-audio-ui-") as directory: + folder = Path(directory) + samples = [] + for i in range(192000): + amplitude = 0 if i < 12000 else (0.15, 0.8, 0.4, 0.65)[(i // 12000) % 4] + samples.append(round(24000 * amplitude * math.sin(i * math.tau * 220 / 48000))) + with wave.open(str(folder / "voice.wav"), "wb") as output: + output.setnchannels(1); output.setsampwidth(2); output.setframerate(48000) + output.writeframes(struct.pack("<" + "h" * len(samples), *samples)) + prepare = folder / "prepare.ts" + prepare.write_text('import { analyzeAudio } from ' + json.dumps(str(ROOT / "audio-metadata.ts")) + ';\n' + 'console.log(JSON.stringify(analyzeAudio(process.argv[2])));\n') + result = subprocess.run(["bun", str(prepare), str(folder / "voice.wav")], capture_output=True, text=True, timeout=30) + if result.returncode: + raise SystemExit("Synthetic audio preparation failed: " + result.stderr[:4096]) + metadata = json.loads(result.stdout) + assert not metadata["unavailable"] and metadata["durationMs"] == 4000 + assert len(metadata["waveform"]) == 64 and max(metadata["waveform"]) == 1 + for file in ["AudioMessage.qml", "AudioPlayback.qml", "AudioWaveform.mjs"]: + shutil.copy(ROOT / file, folder) + capture = "" if args.capture is None else str(args.capture.resolve()) + if capture: + args.capture.resolve().parent.mkdir(parents=True, exist_ok=True) + (folder / "tst_audio.qml").write_text('''import QtQuick +import QtTest +Item { + id: root + width: 350; height: 190 + property int ended: 0 + property int failures: 0 + property var rows: [1] + Repeater { model: root.rows; delegate: AudioMessage { playback: engine; visible: false } } + Rectangle { anchors.fill: parent; color: "#202127" } + AudioPlayback { + id: engine + source: @PLAYBACK@; muted: true; playing: card.playing; resetToken: card.resetToken + onPlaybackEnded: { root.ended++; card.playing = false } + onPlaybackFailed: root.failures++ + } + AudioMessage { + id: card; playback: engine; x: 30; y: 28; width: 288 + source: "voice.wav"; muted: true + waveform: @WAVEFORM@; durationMs: 4000 + onToggleRequested: playing = !playing + } + AudioMessage { + x: 90; y: 112; width: 230; mine: true; muted: true + waveform: @WAVEFORM@; durationMs: 37000 + } + TestCase { + name: "AudioMessagePlayback" + when: windowShown + function initTestCase() { + tryCompare(card, "ready", true, 5000) + compare(card.totalDurationMs, 4000) + compare(root.failures, 0) + } + function test_01_play_pause_resume_and_reset() { + card.playing = true + tryVerify(function() { return card.positionMs > 100 }, 3000) + card.playing = false + wait(120) + var position = card.positionMs + wait(200) + compare(card.positionMs, position) + card.playing = true + tryVerify(function() { return card.positionMs > position + 100 }, 3000) + card.playing = false + card.resetToken++ + tryCompare(card, "positionMs", 0) + verify(!card.actuallyPlaying) + } + function test_02_seek_and_keyboard_focus() { + var slider = findChild(card, "audioTimeline") + verify(slider !== null) + tryCompare(card, "seekable", true) + mouseClick(slider, slider.width / 2, slider.height / 2) + tryVerify(function() { return card.positionMs > 1000 && card.positionMs < 3000 }) + slider.forceActiveFocus() + verify(card.controlsFocused) + keyClick(Qt.Key_Left) + tryCompare(card, "positionMs", 0) + } + function test_03_completion_and_replay() { + var slider = findChild(card, "audioTimeline") + mouseClick(slider, slider.width * 0.75, slider.height / 2) + card.playing = true + tryVerify(function() { return root.ended > 0 }, 5000) + verify(!card.playing) + card.playing = true + tryVerify(function() { return card.positionMs > 0 && card.positionMs < 2000 }, 3000) + card.playing = false + card.resetToken++ + compare(root.failures, 0) + } + function test_04_delegate_rebuild_preserves_playback() { + card.playing = true + tryVerify(function() { return engine.position > 100 }, 3000) + var before = engine.position + root.rows = [2, 3] + tryVerify(function() { return engine.position > before + 100 }, 3000) + card.playing = false + var slider = findChild(card, "audioTimeline") + slider.forceActiveFocus() + keyClick(Qt.Key_Escape) + } + function test_05_compact_layout_and_unavailable_waveform() { + card.width = 160 + verify(card.bars.length <= 20) + card.waveform = [] + compare(card.bars.length, 0) + verify(card.totalDurationMs > 0) + card.width = 288 + card.waveform = @WAVEFORM@ + } + function cleanupTestCase() { + card.playing = false + card.resetToken++ + var path = @CAPTURE@ + if (path !== "") grabImage(root).save(path) + } + } +} +'''.replace("@PLAYBACK@", json.dumps(metadata["playbackUrl"])).replace("@WAVEFORM@", json.dumps(metadata["waveform"])).replace("@CAPTURE@", json.dumps(capture))) + return subprocess.run([runner, "-input", str(folder), "-platform", "offscreen"], timeout=30, + env={**os.environ, "QT_QPA_PLATFORM":"offscreen", "QT_QPA_PLATFORMTHEME":"generic", + "QT_QUICK_BACKEND":"software", "QT_MEDIA_BACKEND":"ffmpeg"}).returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tier2.test.ts b/tier2.test.ts index 64e249ba..a5b62eab 100644 --- a/tier2.test.ts +++ b/tier2.test.ts @@ -1185,3 +1185,19 @@ describe("bridgeRun keeps source routing", () => { } }); }); + + +test("mixed-case audio MIME is analysed on download and cache hit", () => { + const bytes = Buffer.alloc(44); + bytes.write("RIFF", 0); bytes.write("WAVE", 8); + let probes = 0, downloads = 0; + const runner = ((cmd: string) => { + if (cmd === "ffprobe") { probes++; return {status: 1, stdout: ""}; } + downloads++; return {status: 0, stdout: bytes, stderr: ""}; + }) as never; + const first = fetchAttachment("73310001", "voice.wav", "Audio/WAV", runner); + const cached = fetchAttachment("73310001", "voice.wav", "Audio/WAV", runner); + expect(first.ok).toBe(true); expect(cached.ok).toBe(true); + expect(first.audio?.unavailable).toBe(true); expect(cached.audio?.unavailable).toBe(true); + expect(probes).toBe(2); expect(downloads).toBe(1); +}); diff --git a/ui.test.ts b/ui.test.ts index da7ad713..958973be 100644 --- a/ui.test.ts +++ b/ui.test.ts @@ -515,14 +515,15 @@ describe("QML safety invariants", () => { expect(parseTouchpadScrollGain("scroll_gain=0.25\ntouchpad_scroll_gain=0.5")).toBe(0.5); }); - test("a voice message plays without a window, and the chip toggles it", () => { - // xdg-open gave audio to mpv, which opened an empty black video window. - expect(panel).toContain('if (String(root.fetchJobMime || "").indexOf("audio/") === 0) {'); - expect(panel).toContain("root.toggleAudio(String(d.path || \"\"))"); - expect(panel).toContain("exec mpv --no-video --force-window=no --no-terminal --really-quiet -- \"$1\""); - expect(panel).toContain('"sh", path]'); // the path is an argument, never interpolated - expect(panel).toContain("var same = audioPlayer.running && root.playingAudio === path"); - // clicking away stops it: surface closed, another thread, view destroyed + test("inline playback scopes explicit play intent to the conversation", () => { + expect(panel).toContain("AudioMessage {"); + expect(panel).toContain('root.enqueueFetch(att, "audio-play", false)'); + expect(panel).toContain("root.audioPlaying = !root.audioPlaying"); + expect(panel).toContain("String(root.active.chat) === root.pendingAudioChat"); + expect(panel).toContain('root.settleAudioFetch(id, d.ok === true, d.error || "")'); + expect(panel).toContain("resetToken: root.audioResetToken"); + expect(panel).not.toContain("exec mpv"); + expect(panel).toContain('root.settleAudioFetch(root.fetchingId, false, "fetch process ended")'); expect(panel).toContain("if (!surfaceOpen) root.stopAudio()"); expect(panel).toContain("root.stopAudio() // another conversation"); expect(panel).toContain("Component.onDestruction: root.stopAudio()"); @@ -1076,7 +1077,7 @@ test("a thread response taken before a local send or failure cannot replace bubb // the picture (or file chip) itself, through the one shared TapbackPill. test("tapbacks on picture-only messages get a pill on the picture", () => { expect(panel).toContain("component TapbackPill: Rectangle {"); - expect(panel.split("TapbackPill {").length - 1).toBe(4); // text bubble, picture, file chip, standalone sticker + expect(panel.split("TapbackPill {").length - 1).toBe(5); // text bubble, picture, file chip, standalone sticker expect(panel).toContain('readonly property bool pillHere: index === 0 && String(bubbleRow.modelData.text || "") === ""'); expect(panel).toContain("+ (pillHere ? Style.space(12) : 0)"); }); From 6458ce021216299bbdd5ff867285053775cfd3bf Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Tue, 10 Nov 2026 14:47:25 -0800 Subject: [PATCH 2/5] Constrain WAV probing and own external fallback playback --- .github/workflows/ci.yml | 2 ++ BlipView.qml | 11 +++++++++-- audio-metadata.ts | 25 ++++++++++++++----------- audio-waveform.test.ts | 23 +++++++++++++++++++++++ docs/AUDIO.md | 2 ++ scripts/test-audio-session-ui.py | 10 +++++++++- 6 files changed, 59 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c43e067..93aef8a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,8 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: latest + - name: Audio probe regression dependencies + run: sudo apt-get update -qq && sudo apt-get install -y -q ffmpeg - name: unit tests (collector, thread, tier2, ui invariants) run: bun test - name: QML badge regression (synthetic data, no desktop needed) diff --git a/BlipView.qml b/BlipView.qml index f60e5af4..34d15dd7 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -1136,7 +1136,7 @@ FocusScope { root.playingAudio = id root.audioPlaying = true } else if (metadata && metadata.externalDemuxer) { - Quickshell.execDetached(["mpv", "--no-video", "--demuxer=lavf", "--demuxer-lavf-format=" + metadata.externalDemuxer, "--", String(root.attFiles[id])]) + root.playExternalAudio(metadata.externalDemuxer, String(root.attFiles[id])) root.note = "Opened audio externally" } else root.note = "Audio preview unavailable β€” right-click to save the file" } else if (!ok && belongsHere) { @@ -1154,7 +1154,14 @@ FocusScope { root.attFiles = files if (requested) root.note = "Could not play this audio message β€” try again or save the file" } + function playExternalAudio(format, path) { + root.stopAudio() + externalAudio.command = ["mpv", "--no-video", "--demuxer=lavf", "--demuxer-lavf-format=" + format, "--", path] + externalAudio.running = true + } + Process { id: externalAudio } function stopAudio() { + externalAudio.running = false root.audioPlaying = false root.playingAudio = "" root.pendingAudioId = "" @@ -2063,7 +2070,7 @@ FocusScope { // The ordinary Open route remains the fallback when QtMultimedia cannot load. if (String(root.fetchJobMime || "").toLowerCase().indexOf("audio/") === 0) { if (d.audio && d.audio.externalDemuxer) - Quickshell.execDetached(["mpv", "--no-video", "--demuxer=lavf", "--demuxer-lavf-format=" + d.audio.externalDemuxer, "--", String(d.path || "")]) + root.playExternalAudio(d.audio.externalDemuxer, String(d.path || "")) else root.note = "Audio format unavailable β€” right-click to save" } else if (root.openableMime(root.fetchJobMime)) { Quickshell.execDetached(["xdg-open", String(d.url || "")]) diff --git a/audio-metadata.ts b/audio-metadata.ts index 1fec0409..a38ac9ef 100644 --- a/audio-metadata.ts +++ b/audio-metadata.ts @@ -2,10 +2,12 @@ import { spawnSync } from "node:child_process"; import { closeSync, constants, fstatSync, openSync, readSync, utimesSync, writeFileSync, renameSync, unlinkSync } from "node:fs"; import { createHash } from "node:crypto"; +import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; import { AUDIO_ANALYSIS_SECONDS, AUDIO_SAMPLE_RATE, waveformFromPcm16 } from "./audio-waveform"; export interface AudioMetadata { durationMs: number; waveform: number[]; unavailable: boolean; playable: boolean; externalDemuxer?: string; playbackUrl?: string } +const PCM_OFFSET = 4096; const MAX_INPUT = 100 * 1024 * 1024; /** Explicit demuxers prevent disguised playlists from loading referenced @@ -42,16 +44,16 @@ export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { if (!n) return empty; hash.update(chunk.subarray(0, n)); offset += n; } - const playbackPath = path + `.pcm-${hash.digest("hex").slice(0, 24)}.wav`; + const playbackPath = join(dirname(path), `.pcm2-${hash.digest("hex")}.wav`); try { const cached = openSync(playbackPath, constants.O_RDONLY | constants.O_NOFOLLOW); try { const cs = fstatSync(cached); - if (cs.isFile() && cs.uid === process.getuid() && cs.size > 44 && cs.size <= AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + 44) { + if (cs.isFile() && cs.uid === process.getuid() && cs.size > PCM_OFFSET && cs.size <= AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + PCM_OFFSET) { const data = Buffer.alloc(cs.size); if (readSync(cached, data, 0, data.length, 0) === data.length && data.toString("ascii", 0, 4) === "RIFF" && data.toString("ascii", 8, 16) === "WAVEfmt ") { const now = new Date(); utimesSync(playbackPath, now, now); - return {durationMs: Math.round((data.length - 44) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(44)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; + return {durationMs: Math.round((data.length - PCM_OFFSET) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(PCM_OFFSET)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; } } } finally { closeSync(cached); } @@ -71,25 +73,26 @@ export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { const seconds = Number(data.format?.duration); if (!Number.isFinite(seconds) || seconds <= 0 || seconds > 86400) return { ...empty, playable: false }; const durationMs = Math.round(seconds * 1000); - fallback = { durationMs, waveform: [], unavailable: true, playable: false }; + fallback = { ...empty, durationMs }; // A waveform must describe the complete recording. Don't display a // truncated ten-minute envelope against a longer recording's timeline. - if (seconds > AUDIO_ANALYSIS_SECONDS) return { durationMs, waveform: [], unavailable: true, playable: false }; + if (seconds > AUDIO_ANALYSIS_SECONDS) return { ...empty, durationMs }; const result = runner("ffmpeg", ["-nostdin", "-v", "error", "-threads", "1", ...input, "-map", "0:a:0", "-vn", "-sn", "-dn", "-t", String(AUDIO_ANALYSIS_SECONDS), "-ac", "1", "-ar", String(AUDIO_SAMPLE_RATE), "-c:a", "pcm_s16le", "-f", "s16le", "pipe:1"], { timeout: 15000, maxBuffer: AUDIO_SAMPLE_RATE * AUDIO_ANALYSIS_SECONDS * 2 + 16384 }); - if (result.status !== 0 || !Buffer.isBuffer(result.stdout)) return { durationMs, waveform: [], unavailable: true, playable: false }; + if (result.status !== 0 || !Buffer.isBuffer(result.stdout)) return { ...empty, durationMs }; const decodedSeconds = result.stdout.length / (2 * AUDIO_SAMPLE_RATE); if (Math.abs(decodedSeconds - seconds) > Math.max(0.25, seconds * 0.02)) - return { durationMs, waveform: [], unavailable: true, playable: false }; + return { ...empty, durationMs }; const waveform = waveformFromPcm16(result.stdout); if (!waveform.length) return empty; - const wav = Buffer.alloc(44); - wav.write("RIFF", 0); wav.writeUInt32LE(result.stdout.length + 36, 4); wav.write("WAVEfmt ", 8); - wav.writeUInt32LE(16, 16); wav.writeUInt16LE(1, 20); wav.writeUInt16LE(1, 22); + const wav = Buffer.alloc(PCM_OFFSET); + wav.write("RIFF", 0); wav.writeUInt32LE(result.stdout.length + PCM_OFFSET - 8, 4); wav.write("WAVEfmt ", 8); + wav.writeUInt32LE(18, 16); wav.writeUInt16LE(1, 20); wav.writeUInt16LE(1, 22); wav.writeUInt32LE(AUDIO_SAMPLE_RATE, 24); wav.writeUInt32LE(AUDIO_SAMPLE_RATE * 2, 28); - wav.writeUInt16LE(2, 32); wav.writeUInt16LE(16, 34); wav.write("data", 36); wav.writeUInt32LE(result.stdout.length, 40); + wav.writeUInt16LE(2, 32); wav.writeUInt16LE(16, 34); wav.write("JUNK", 38); wav.writeUInt32LE(PCM_OFFSET - 54, 42); + wav.write("data", PCM_OFFSET - 8); wav.writeUInt32LE(result.stdout.length, PCM_OFFSET - 4); const tmp = playbackPath + `.tmp-${process.pid}`; try { writeFileSync(tmp, Buffer.concat([wav, result.stdout]), {mode:0o600, flag:"wx"}); diff --git a/audio-waveform.test.ts b/audio-waveform.test.ts index 513717d8..018b15e2 100644 --- a/audio-waveform.test.ts +++ b/audio-waveform.test.ts @@ -1,6 +1,8 @@ import { describe, expect, test } from "bun:test"; import { mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import { join } from "node:path"; import { AUDIO_ANALYSIS_SECONDS, AUDIO_SAMPLE_RATE, durationLabel, fitWaveform, normalizeWaveform, playbackFraction, seekPosition, waveformFromPcm16 } from "./audio-waveform"; @@ -85,6 +87,7 @@ describe("bounded private audio analysis", () => { const r = analyzeAudio(path, (() => { calls++; return {status:0,stdout:JSON.stringify({streams:[{codec_type:"audio"}],format:{duration:seconds}})}; }) as never); expect(calls).toBe(1); expect(r.waveform).toEqual([]); expect(r.unavailable).toBe(true); expect(r.durationMs).toBe(seconds===601 ? 601000 : 0); + expect(r.externalDemuxer).toBe("wav"); } const timedOut = analyzeAudio(path, ((cmd: string) => cmd === "ffprobe" ? {status:0,stdout:'{"streams":[{"codec_type":"audio"}],"format":{"duration":"2"}}'} : {status:null,stdout:Buffer.alloc(0),error:Error("ETIMEDOUT")}) as never); @@ -111,3 +114,23 @@ test("a lying duration never produces playable normalized audio", () => { expect(r.playable).toBe(false); expect(r.playbackUrl).toBeUndefined(); } finally {rmSync(dir,{recursive:true,force:true});} }); + + +test("normalized sender PCM cannot override WAV format probing", () => { + const dir = mkdtempSync(join(tmpdir(), "blip-audio-probe-")); + try { + const ts = spawnSync("ffmpeg", ["-nostdin", "-v", "error", "-f", "lavfi", "-i", "testsrc=size=32x32:rate=25", "-f", "lavfi", "-i", "sine=frequency=440", "-t", "0.5", "-c:v", "mpeg2video", "-c:a", "mp2", "-f", "mpegts", "pipe:1"]); + expect(ts.status).toBe(0); + const payload = Buffer.alloc(Math.ceil(ts.stdout.length / 2) * 2); ts.stdout.copy(payload); + const input = join(dir, "voice.wav"); writeFileSync(input, wavHeader()); + const r = analyzeAudio(input, ((cmd: string) => cmd === "ffprobe" + ? {status:0, stdout:JSON.stringify({streams:[{codec_type:"audio"}],format:{duration:payload.length / (2 * AUDIO_SAMPLE_RATE)}})} + : {status:0, stdout:payload}) as never); + expect(r.playable).toBe(true); + const guessed = spawnSync("ffprobe", ["-v","error","-show_entries","format=format_name:stream=codec_type","-of","json",fileURLToPath(r.playbackUrl!)]); + expect(guessed.status).toBe(0); + const meta = JSON.parse(String(guessed.stdout)); + expect(meta.format.format_name).toBe("wav"); + expect(meta.streams.map((s:any)=>s.codec_type)).toEqual(["audio"]); + } finally { rmSync(dir,{recursive:true,force:true}); } +}); diff --git a/docs/AUDIO.md b/docs/AUDIO.md index 6884233b..0e893069 100644 --- a/docs/AUDIO.md +++ b/docs/AUDIO.md @@ -39,3 +39,5 @@ python3 scripts/test-audio-ui.py ![Synthetic inline players](img/inline-audio.png) Analysis happens only on a play click, never on thread open. Qt opens only the generated 24 kHz mono PCM WAV, never the received container. This bounds playback to ten minutes; unavailable normalization and missing tools fall back to external mpv or Save. The normalized file also caches the waveform by source content hash. The view owns the optional multimedia Loader independently of message delegates; model reloads preserve playback, leaving the conversation or closing the view stops it. Missing QtMultimedia falls back to an ordinary attachment chip. + +Generated WAVs put their data at offset 4096 behind an extended format chunk and padding, so initial format probing sees only generated bytes. Versioned, content-hash cache names invalidate the previous 44-byte-header format and avoid long source filenames. External mpv is owned by the view and stopped with inline playback. diff --git a/scripts/test-audio-session-ui.py b/scripts/test-audio-session-ui.py index 67668de0..7b42a7af 100644 --- a/scripts/test-audio-session-ui.py +++ b/scripts/test-audio-session-ui.py @@ -24,6 +24,7 @@ def function(name): import QtTest Item { id: root + property QtObject externalAudio: QtObject { property bool running: false; property var command: [] } property var audioPlayback: ({status: Loader.Ready}) property int rowBudget: 1 property int rowBatch: 1 @@ -111,12 +112,19 @@ def function(name): root.openAttachment({id:"b",mime:"Audio/WAV"}); compare(root.requests[0].action,"open"); } + function test_external_fallback_stops_on_surface_close() { + root.toggleAudio({id:"a"}); root.attFiles=({a:"file:///synthetic/voice.ogg"}); + root.attMetrics=({a:{audio:{playable:false,externalDemuxer:"ogg"}}}); + root.settleAudioFetch("a",true); + verify(externalAudio.running); verify(externalAudio.command.indexOf("--") >= 0); + root.surfaceOpen=false; verify(!externalAudio.running); + } function test_missing_id_or_closed_surface_cannot_request_play() { root.toggleAudio({id:""}); root.surfaceOpen=false; root.toggleAudio({id:"a"}); compare(root.requests.length,0); } } } -'''.replace('@SURFACE@', source[source.index(' onSurfaceOpenChanged: {'):source.index(' function pumpAvatar()', source.index(' onSurfaceOpenChanged: {'))]).replace('@FUNCTIONS@', '\n'.join(function(name) for name in ['toggleAudio','settleAudioFetch','failAudio','stopAudio','openAttachment'])) +'''.replace('@SURFACE@', source[source.index(' onSurfaceOpenChanged: {'):source.index(' function pumpAvatar()', source.index(' onSurfaceOpenChanged: {'))]).replace('@FUNCTIONS@', '\n'.join(function(name) for name in ['toggleAudio','settleAudioFetch','failAudio','stopAudio','openAttachment','playExternalAudio'])) runner = shutil.which('qmltestrunner') or '/usr/lib/qt6/bin/qmltestrunner' with tempfile.TemporaryDirectory(prefix='blip-audio-session-') as tmp: path=Path(tmp); (path/'tst_audio.qml').write_text(qml) From c0a16a239e2301e58c525eefcf9bcfcbc2ef19ed Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Tue, 10 Nov 2026 14:47:44 -0800 Subject: [PATCH 3/5] Start fallback on the first missing multimedia load --- BlipView.qml | 8 ++++++++ audio-metadata.ts | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/BlipView.qml b/BlipView.qml index 34d15dd7..b7c1d0a9 100644 --- a/BlipView.qml +++ b/BlipView.qml @@ -1097,6 +1097,14 @@ FocusScope { id: audioPlayback active: root.playingAudio !== "" source: "AudioPlayback.qml" + onStatusChanged: { + if (status === Loader.Error && root.playingAudio !== "") { + var id = root.playingAudio + var metadata = (root.attMetrics[id] || {}).audio + if (metadata && metadata.externalDemuxer) root.playExternalAudio(metadata.externalDemuxer, String(root.attFiles[id] || "")) + else { root.stopAudio(); root.note = "Audio preview unavailable β€” right-click to save" } + } + } onLoaded: { item.source = Qt.binding(function() { return ((root.attMetrics[root.playingAudio] || {}).audio || {}).playbackUrl || "" }) item.playing = Qt.binding(function() { return root.audioPlaying }) diff --git a/audio-metadata.ts b/audio-metadata.ts index a38ac9ef..dcee81d8 100644 --- a/audio-metadata.ts +++ b/audio-metadata.ts @@ -53,7 +53,7 @@ export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { const data = Buffer.alloc(cs.size); if (readSync(cached, data, 0, data.length, 0) === data.length && data.toString("ascii", 0, 4) === "RIFF" && data.toString("ascii", 8, 16) === "WAVEfmt ") { const now = new Date(); utimesSync(playbackPath, now, now); - return {durationMs: Math.round((data.length - PCM_OFFSET) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(PCM_OFFSET)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; + return {externalDemuxer: audioDemuxer(chunk.subarray(0, Math.min(16, st.size))), durationMs: Math.round((data.length - PCM_OFFSET) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(PCM_OFFSET)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; } } } finally { closeSync(cached); } @@ -98,7 +98,7 @@ export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { writeFileSync(tmp, Buffer.concat([wav, result.stdout]), {mode:0o600, flag:"wx"}); renameSync(tmp, playbackPath); } finally { try { unlinkSync(tmp); } catch {} } - return { durationMs, waveform, unavailable: false, playable: true, playbackUrl:pathToFileURL(playbackPath).href }; + return { durationMs, waveform, unavailable: false, playable: true, externalDemuxer: format, playbackUrl:pathToFileURL(playbackPath).href }; } catch { return fallback; } finally { if (fd !== undefined) closeSync(fd); } } From c922fc7b5ed51d1ec2515431108ac83c3d55415a Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Tue, 10 Nov 2026 14:48:06 -0800 Subject: [PATCH 4/5] Read source format before looking up normalized audio --- audio-metadata.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/audio-metadata.ts b/audio-metadata.ts index dcee81d8..eb05bbd1 100644 --- a/audio-metadata.ts +++ b/audio-metadata.ts @@ -36,6 +36,10 @@ export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW); const st = fstatSync(fd); if (!st.isFile() || st.uid !== process.getuid() || st.size <= 0 || st.size > MAX_INPUT) return empty; + const header = Buffer.alloc(16); const length = readSync(fd, header, 0, header.length, 0); + const format = audioDemuxer(header.subarray(0, length)); + if (!format) return empty; + empty.externalDemuxer = format; // Generated PCM WAV is the only file ever handed to Qt. The source // fingerprint invalidates both waveform and normalized audio together. const hash = createHash("sha256"), chunk = Buffer.alloc(65536); @@ -53,15 +57,11 @@ export function analyzeAudio(path: string, runner = spawnSync): AudioMetadata { const data = Buffer.alloc(cs.size); if (readSync(cached, data, 0, data.length, 0) === data.length && data.toString("ascii", 0, 4) === "RIFF" && data.toString("ascii", 8, 16) === "WAVEfmt ") { const now = new Date(); utimesSync(playbackPath, now, now); - return {externalDemuxer: audioDemuxer(chunk.subarray(0, Math.min(16, st.size))), durationMs: Math.round((data.length - PCM_OFFSET) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(PCM_OFFSET)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; + return {externalDemuxer: format, durationMs: Math.round((data.length - PCM_OFFSET) * 1000 / (2 * AUDIO_SAMPLE_RATE)), waveform: waveformFromPcm16(data.subarray(PCM_OFFSET)), unavailable:false, playable:true, playbackUrl:pathToFileURL(playbackPath).href}; } } } finally { closeSync(cached); } } catch { /* no normalized cache entry */ } - const header = Buffer.alloc(16); const length = readSync(fd, header, 0, header.length, 0); - const format = audioDemuxer(header.subarray(0, length)); - if (!format) return empty; - empty.externalDemuxer = format; const input = ["-protocol_whitelist", "file,pipe", "-f", format, "-i", path]; const probe = runner("ffprobe", ["-v", "error", ...input, "-select_streams", "a:0", "-show_entries", "format=duration:stream=codec_type", "-of", "json"], From 0a20ad5f5c8ee1650b9580e185a768b6c689d1fd Mon Sep 17 00:00:00 2001 From: Marcus Vorwaller Date: Tue, 10 Nov 2026 14:48:28 -0800 Subject: [PATCH 5/5] Exercise missing multimedia fallback through the shipping loader --- .github/workflows/ci.yml | 1 + scripts/test-audio-loader.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 scripts/test-audio-loader.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93aef8a0..5deed13b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,7 @@ jobs: sudo apt-get install -y -q ffmpeg qml6-module-qtmultimedia qml6-module-qtquick-controls qml6-module-qtquick-templates python3 scripts/test-audio-ui.py python3 scripts/test-audio-session-ui.py + python3 scripts/test-audio-loader.py - name: OTP desktop adapter (no desktop needed) run: | python3 -m py_compile otp-desktop.py diff --git a/scripts/test-audio-loader.py b/scripts/test-audio-loader.py new file mode 100644 index 00000000..73ef46a7 --- /dev/null +++ b/scripts/test-audio-loader.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Exercise the shipping optional player loader with a missing module.""" +from pathlib import Path +import tempfile, subprocess, os +source=Path(__file__).resolve().parents[1].joinpath('BlipView.qml').read_text() +at=source.index(' Loader {\n id: audioPlayback') +begin=source.index('{',at);depth=0 +for end in range(begin,len(source)): + if source[end]=='{':depth+=1 + elif source[end]=='}': + depth-=1 + if depth==0:break +loader=source[at:end+1] +qml='''import QtQuick +import QtTest +Item { + id: root + property string playingAudio: "a" + property var attFiles: ({a:"file:///synthetic/voice.ogg"}) + property var attMetrics: ({a:{audio:{playable:true,externalDemuxer:"ogg"}}}) + property bool audioPlaying: true + property bool audioMuted: true + property int audioResetToken: 0 + property string note: "" + property bool fallback: false + function stopAudio() { playingAudio="" } + function playExternalAudio(format,path) { fallback=format==="ogg" && path===attFiles.a; stopAudio() } + @LOADER@ + TestCase { name: "OptionalAudioLoader" + function test_missing_module_falls_back_on_first_play() { tryCompare(root,"fallback",true); compare(root.playingAudio,"") } + } +}'''.replace('@LOADER@',loader) +with tempfile.TemporaryDirectory(prefix='blip-audio-loader-') as tmp: + Path(tmp,'AudioPlayback.qml').write_text('import QtQuick\nimport MissingMultimedia\nItem {}') + Path(tmp,'tst_loader.qml').write_text(qml) + raise SystemExit(subprocess.run(['/usr/lib/qt6/bin/qmltestrunner','-input',tmp],env={**os.environ,'QT_QPA_PLATFORM':'offscreen','QT_QUICK_BACKEND':'software','QT_QPA_PLATFORMTHEME':'generic'}).returncode)