From 51273b60062d4699adb6dfa43c7b4fdf1e9a8f72 Mon Sep 17 00:00:00 2001 From: ilgiova237 Date: Wed, 18 Nov 2026 20:01:43 +0100 Subject: [PATCH 1/3] perf(audio): defer idle DSP setup until decoding --- .github/workflows/linux-dbus-check.yml | 2 +- build.gradle.kts | 2 +- .../alananasss/kittytune/audio/AudioEngine.kt | 202 ++++++++++++------ src/test/kotlin/LazyAudioEngineTest.kt | 126 +++++++++++ 4 files changed, 265 insertions(+), 67 deletions(-) create mode 100644 src/test/kotlin/LazyAudioEngineTest.kt diff --git a/.github/workflows/linux-dbus-check.yml b/.github/workflows/linux-dbus-check.yml index 7c4af367..bd4e62e3 100644 --- a/.github/workflows/linux-dbus-check.yml +++ b/.github/workflows/linux-dbus-check.yml @@ -34,7 +34,7 @@ jobs: sudo apt-get -o Acquire::Retries=3 install -y fakeroot python3-dbus python3-gi dbus-x11 xvfb xauth playerctl libasound2t64 libgl1 libegl1 - name: Test D-Bus services and build release package run: | - dbus-run-session -- bash gradlew :test --tests '*MprisServiceTest*' --tests '*LinuxStatusNotifierServiceTest*' -PdesktopOnly=true --console=plain + dbus-run-session -- bash gradlew :test --tests '*MprisServiceTest*' --tests '*LinuxStatusNotifierServiceTest*' --tests '*LazyAudioEngineTest*' -PdesktopOnly=true --console=plain bash gradlew packageReleaseDeb -PdesktopOnly=true --console=plain - name: Exercise final Debian package with its embedded runtime run: | diff --git a/build.gradle.kts b/build.gradle.kts index c7b7b508..8c1568da 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1004,7 +1004,7 @@ tasks.register("headlessTest") { "OnnxNativeLoadTest", "ToolWindowStyleRaiseToFrontTest", "*SoundCloudOAuthTest", "ClearStreamFilesTest", "ThemePersistenceTest", "SetupOnboardingTest", "*WindowsProtectedPlaybackTest", "*WindowsProtectedMediaTest", "Issue93BugfixesTest.testWindowsUpdaterScriptConstruction*", "InstallerProcessTest", "*ChromeCdmArchiveTest", "KittyTuneProcessesTest", "*SidecarLockOrderTest", - "*HlsOpeningTest", "*WindowsHeadphoneDetectionTest" + "*HlsOpeningTest", "*WindowsHeadphoneDetectionTest", "*LazyAudioEngineTest" ).forEach(::includeTestsMatching) } } diff --git a/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt b/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt index 7c832d11..e4434c1e 100644 --- a/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt +++ b/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt @@ -76,39 +76,76 @@ class AudioEngine(private val isolated: Boolean = false) { private val outputSampleRate = 44100 private val outputChannels = 2 - private val fx = FxAudioProcessor() - private val equalizer = EqualizerAudioProcessor() - private val reverb = ReverbAudioProcessor() - private val eightD = EightDAudioProcessor() - private val earrape = EarrapeAudioProcessor() - private val normalization = NormalizationAudioProcessor() - private val mono = MonoAudioProcessor() - private val vintageMp3 = VintageMp3AudioProcessor() - private val vocalRemover = VocalRemoverAudioProcessor() - private val vocalBoost = VocalBoostAudioProcessor() - private val flanger = FlangerAudioProcessor() - private val partyNextDoor = PartyNextDoorAudioProcessor() - private val superWide = SuperWideAudioProcessor() - private val vinylLoFi = VinylLoFiAudioProcessor() - private val phaser = PhaserAudioProcessor() - private val megaphone = MegaphoneRadioAudioProcessor() - private val robotVocoder = RobotVocoderAudioProcessor() - private val chorus = ChorusAudioProcessor() - private val underwater = UnderwaterAudioProcessor() - private val tranceGate = TranceGateAudioProcessor() - private val pingPongDelay = PingPongDelayAudioProcessor() - private val chiptune = ChiptuneAudioProcessor() - private val shimmerReverb = ShimmerReverbAudioProcessor() - private val rotarySpeaker = RotarySpeakerAudioProcessor() - private val tapeSaturation = TapeSaturationAudioProcessor() - private val subOctaver = SubOctaverAudioProcessor() - private val emptyMall = EmptyMallAudioProcessor() - private val gramophone = GramophoneAudioProcessor() - private val reverseEcho = ReverseEchoAudioProcessor() - private val stadium = StadiumAudioProcessor() - private val walkman = CassetteWalkmanAudioProcessor() - private val asmrVocal = AsmrVocalAudioProcessor() - private val nightDrive = NightDriveAudioProcessor() + private class EffectNodes { + val fx = FxAudioProcessor() + val equalizer = EqualizerAudioProcessor() + val reverb = ReverbAudioProcessor() + val eightD = EightDAudioProcessor() + val earrape = EarrapeAudioProcessor() + val normalization = NormalizationAudioProcessor() + val mono = MonoAudioProcessor() + val vintageMp3 = VintageMp3AudioProcessor() + val vocalRemover = VocalRemoverAudioProcessor() + val vocalBoost = VocalBoostAudioProcessor() + val flanger = FlangerAudioProcessor() + val partyNextDoor = PartyNextDoorAudioProcessor() + val superWide = SuperWideAudioProcessor() + val vinylLoFi = VinylLoFiAudioProcessor() + val phaser = PhaserAudioProcessor() + val megaphone = MegaphoneRadioAudioProcessor() + val robotVocoder = RobotVocoderAudioProcessor() + val chorus = ChorusAudioProcessor() + val underwater = UnderwaterAudioProcessor() + val tranceGate = TranceGateAudioProcessor() + val pingPongDelay = PingPongDelayAudioProcessor() + val chiptune = ChiptuneAudioProcessor() + val shimmerReverb = ShimmerReverbAudioProcessor() + val rotarySpeaker = RotarySpeakerAudioProcessor() + val tapeSaturation = TapeSaturationAudioProcessor() + val subOctaver = SubOctaverAudioProcessor() + val emptyMall = EmptyMallAudioProcessor() + val gramophone = GramophoneAudioProcessor() + val reverseEcho = ReverseEchoAudioProcessor() + val stadium = StadiumAudioProcessor() + val walkman = CassetteWalkmanAudioProcessor() + val asmrVocal = AsmrVocalAudioProcessor() + val nightDrive = NightDriveAudioProcessor() + } + + private val effectNodes = lazy { EffectNodes() } + private val fx get() = effectNodes.value.fx + private val equalizer get() = effectNodes.value.equalizer + private val reverb get() = effectNodes.value.reverb + private val eightD get() = effectNodes.value.eightD + private val earrape get() = effectNodes.value.earrape + private val normalization get() = effectNodes.value.normalization + private val mono get() = effectNodes.value.mono + private val vintageMp3 get() = effectNodes.value.vintageMp3 + private val vocalRemover get() = effectNodes.value.vocalRemover + private val vocalBoost get() = effectNodes.value.vocalBoost + private val flanger get() = effectNodes.value.flanger + private val partyNextDoor get() = effectNodes.value.partyNextDoor + private val superWide get() = effectNodes.value.superWide + private val vinylLoFi get() = effectNodes.value.vinylLoFi + private val phaser get() = effectNodes.value.phaser + private val megaphone get() = effectNodes.value.megaphone + private val robotVocoder get() = effectNodes.value.robotVocoder + private val chorus get() = effectNodes.value.chorus + private val underwater get() = effectNodes.value.underwater + private val tranceGate get() = effectNodes.value.tranceGate + private val pingPongDelay get() = effectNodes.value.pingPongDelay + private val chiptune get() = effectNodes.value.chiptune + private val shimmerReverb get() = effectNodes.value.shimmerReverb + private val rotarySpeaker get() = effectNodes.value.rotarySpeaker + private val tapeSaturation get() = effectNodes.value.tapeSaturation + private val subOctaver get() = effectNodes.value.subOctaver + private val emptyMall get() = effectNodes.value.emptyMall + private val gramophone get() = effectNodes.value.gramophone + private val reverseEcho get() = effectNodes.value.reverseEcho + private val stadium get() = effectNodes.value.stadium + private val walkman get() = effectNodes.value.walkman + private val asmrVocal get() = effectNodes.value.asmrVocal + private val nightDrive get() = effectNodes.value.nightDrive val automixDuck = com.alananasss.kittytune.audio.automix.AutomixDuckAudioProcessor() /** The Mix transitions' sweep filter (RISE high-pass, WAVE band-pass, a Custom seam's filter legs). */ @@ -141,30 +178,51 @@ class AudioEngine(private val isolated: Boolean = false) { */ val silenceSkip = com.alananasss.kittytune.audio.silence.SilenceSkipAudioProcessor() - private val chain: List = listOfNotNull( - aiDetection.takeUnless { isolated }, - liveBeatTap, - silenceSkip, - automixDuck, - mixSweep, - transitionSweep, - transitionEffects, - fx, equalizer, reverb, eightD, earrape, normalization, mono, - vintageMp3, vocalRemover, vocalBoost, flanger, partyNextDoor, - superWide, vinylLoFi, phaser, megaphone, robotVocoder, chorus, - underwater, tranceGate, pingPongDelay, chiptune, shimmerReverb, - rotarySpeaker, tapeSaturation, subOctaver, emptyMall, gramophone, - reverseEcho, stadium, walkman, asmrVocal, nightDrive - ) - private val stretcher = TimeStretcher(outputSampleRate, outputChannels) + // An empty list marks release, including a graph still being configured on the decoder. + private val initializedDsp = AtomicReference?>(null) + private val chain: List by lazy { + listOfNotNull( + aiDetection.takeUnless { isolated }, + liveBeatTap, + silenceSkip, + automixDuck, + mixSweep, + transitionSweep, + transitionEffects, + fx, equalizer, reverb, eightD, earrape, normalization, mono, + vintageMp3, vocalRemover, vocalBoost, flanger, partyNextDoor, + superWide, vinylLoFi, phaser, megaphone, robotVocoder, chorus, + underwater, tranceGate, pingPongDelay, chiptune, shimmerReverb, + rotarySpeaker, tapeSaturation, subOctaver, emptyMall, gramophone, + reverseEcho, stadium, walkman, asmrVocal, nightDrive + ).also { processors -> + try { + val format = AudioFormat(outputSampleRate, outputChannels) + processors.forEach { it.configure(format) } + if (!initializedDsp.compareAndSet(null, processors)) { + throw InterruptedException("Audio engine released while preparing DSP") + } + } catch (failure: Throwable) { + processors.forEach { runCatching { it.reset() } } + throw failure + } + } + } + private val stretcherState = lazy { TimeStretcher(outputSampleRate, outputChannels) } + private val stretcher get() = stretcherState.value /** * Catches the inter-sample overshoot lossy decoders leave above full scale, before * the 16-bit conversion turns it into flat-topped crackle. See [PeakLimiter]. */ - private val limiter = PeakLimiter(outputSampleRate, outputChannels) + private val limiterState = lazy { PeakLimiter(outputSampleRate, outputChannels) } + private val limiter get() = limiterState.value + @Volatile private var effects = AudioEffectsState() + private val pendingEffects = AtomicReference(effects) + private val pendingEqualizer = AtomicReference(null) + private val pendingStretch = AtomicReference?>(1f to 1f) @Volatile private var volume: Float = 1f @@ -226,8 +284,6 @@ class AudioEngine(private val isolated: Boolean = false) { private var sourceAnalysisPending = false init { - val fmt = AudioFormat(outputSampleRate, outputChannels) - chain.forEach { it.configure(fmt) } aiDetection.setPreferences(PlayerPreferences()) if (!isolated && !com.alananasss.kittytune.audio.ai.AiDetectionManager.isModelLoaded() && com.alananasss.kittytune.audio.ai.AiDetectionManager.isModelReady() && @@ -509,7 +565,7 @@ class AudioEngine(private val isolated: Boolean = false) { val l = line ?: return positionMs val bytesPerSecond = outputSampleRate * outputChannels * 2 val queuedBytes = runCatching { (l.bufferSize - l.available()).coerceAtLeast(0) }.getOrDefault(0) - val queuedSamples = queuedBytes / 2 + runCatching { stretcher.available() }.getOrDefault(0) + val queuedSamples = queuedBytes / 2 + if (stretcherState.isInitialized()) stretcher.available() else 0 val queuedMs = queuedSamples * 1000L / (bytesPerSecond / 2) return (positionMs - queuedMs).coerceAtLeast(0L) } @@ -557,7 +613,7 @@ class AudioEngine(private val isolated: Boolean = false) { // chain, which keeps native DSP state (normalisation handle, ebur128 histograms) // allocated. Every crossfade releases an engine, so without this reset each // played track leaked ~0.4 MB of native memory (issue #66: 500 MB -> 1 GB+). - chain.forEach { runCatching { it.reset() } } + initializedDsp.getAndSet(emptyList())?.forEach { runCatching { it.reset() } } scope.coroutineContext[Job]?.cancel() com.alananasss.kittytune.utils.NativeMemoryManager.trim() } @@ -579,6 +635,11 @@ class AudioEngine(private val isolated: Boolean = false) { fun applyEffects(state: AudioEffectsState) { effects = state + pendingEffects.set(state) + pendingStretch.set(state.speed to if (state.isPitchEnabled) state.speed else 1f) + } + + private fun applyEffectsNow(state: AudioEffectsState) { fx.setEffects(state.isMuffledEnabled, state.isBassBoostEnabled) fx.setBassBoostGain(state.bassBoostIntensity) fx.setMuffledCutoff(state.muffledIntensity) @@ -698,9 +759,6 @@ class AudioEngine(private val isolated: Boolean = false) { nightDrive.setCabinWidth(state.nightDriveCabin) nightDrive.setRoadRumble(state.nightDriveRoad) - val pitch = if (state.isPitchEnabled) state.speed else 1f - stretcher.setParameters(state.speed, pitch) - limiter.setCeiling(peakLimiterCeilingFor(state)) } @@ -731,6 +789,10 @@ class AudioEngine(private val isolated: Boolean = false) { fun applyEqualizer(state: EqualizerState) { equalizerEnabled = state.isEnabled + pendingEqualizer.set(state.copy(bandGainsDb = state.bandGainsDb.toList())) + } + + private fun applyEqualizerNow(state: EqualizerState) { equalizer.setParameters( enabled = state.isEnabled, preampDb = state.preampDb, @@ -762,7 +824,7 @@ class AudioEngine(private val isolated: Boolean = false) { val basePitch = if (effects.isPitchEnabled) baseSpeed else 1f val adjSpeed = (baseSpeed * tempoRatio).coerceIn(0.5f, 2.0f) val adjPitch = (basePitch * pitchRatio).coerceIn(0.5f, 2.0f) - stretcher.setParameters(adjSpeed, adjPitch) + pendingStretch.set(adjSpeed to adjPitch) } @@ -1137,9 +1199,9 @@ class AudioEngine(private val isolated: Boolean = false) { aiDetection.notifySeek(seek) vocalRemover.seekToPositionMs(seek) vocalBoost.seekToPositionMs(seek) - stretcher.flush() - limiter.flush() - chain.forEach { it.flush() } + if (stretcherState.isInitialized()) stretcher.flush() + if (limiterState.isInitialized()) limiter.flush() + initializedDsp.get()?.forEach { it.flush() } liveBeat.onFlush(seek.toDouble()) localLine?.flush() if (!seekRequest.compareAndSet(request, null)) continue @@ -1239,6 +1301,7 @@ class AudioEngine(private val isolated: Boolean = false) { val n = decodeToFloat(buffers, frameChannels) if (n <= 0) return + configuredProcessors() limiter.process(decodeBuffer, n) if (interleaveBuffer.size < n) interleaveBuffer = ShortArray(n) @@ -1302,6 +1365,15 @@ class AudioEngine(private val isolated: Boolean = false) { private var dspInputBuffer = ByteBuffer.allocateDirect(0).order(ByteOrder.LITTLE_ENDIAN) private var processedBuffer = ShortArray(0) + private fun configuredProcessors(): List { + if (released) throw InterruptedException("Audio engine released") + val processors = chain + pendingEffects.getAndSet(null)?.let(::applyEffectsNow) + pendingEqualizer.getAndSet(null)?.let(::applyEqualizerNow) + pendingStretch.getAndSet(null)?.let { (speed, pitch) -> stretcher.setParameters(speed, pitch) } + return processors + } + private fun pushThroughDsp(pcm: ShortArray, length: Int = pcm.size) { val requiredBytes = length * 2 var buf = if (dspInputBuffer.capacity() < requiredBytes) { @@ -1315,7 +1387,7 @@ class AudioEngine(private val isolated: Boolean = false) { buf.position(0) buf.limit(requiredBytes) - for (p in chain) { + for (p in configuredProcessors()) { p.queueInput(buf) buf = p.getOutput() } @@ -1440,9 +1512,9 @@ class AudioEngine(private val isolated: Boolean = false) { worker = null reapFinishedWorkers() silenceAndCloseLater(line) - stretcher.flush() - limiter.flush() - chain.forEach { it.flush() } + if (stretcherState.isInitialized()) stretcher.flush() + if (limiterState.isInitialized()) limiter.flush() + initializedDsp.get()?.forEach { it.flush() } setPlaying(false) } diff --git a/src/test/kotlin/LazyAudioEngineTest.kt b/src/test/kotlin/LazyAudioEngineTest.kt new file mode 100644 index 00000000..1c38e9ca --- /dev/null +++ b/src/test/kotlin/LazyAudioEngineTest.kt @@ -0,0 +1,126 @@ +import com.alananasss.kittytune.audio.AudioEngine +import com.alananasss.kittytune.audio.AudioProcessor +import com.alananasss.kittytune.audio.TimeStretcher +import com.alananasss.kittytune.ui.player.AudioEffectsState +import com.alananasss.kittytune.ui.player.EqualizerState +import org.junit.Test +import kotlin.test.* + +class LazyAudioEngineTest { + private val deferredNames = setOf("fx", "equalizer", "normalization", "reverb", "vocalRemover", "vocalBoost", "stretcher") + + private fun initialized(engine: AudioEngine): Set = engine.javaClass.declaredFields.mapNotNull { field -> + val name = field.name.removeSuffix("\$delegate").removeSuffix("State") + if (name !in deferredNames) return@mapNotNull null + field.isAccessible = true + when (val value = field.get(engine)) { + is Lazy<*> -> name.takeIf { value.isInitialized() } + is AudioProcessor, is TimeStretcher -> name + else -> null + } + }.toSet() + engine.javaClass.declaredFields.filter { it.name == "effectNodes" }.flatMap { field -> + field.isAccessible = true + val holder = field.get(engine) + val nodes = if (holder is Lazy<*>) { + if (holder.isInitialized()) holder.value else null + } else holder + nodes?.javaClass?.declaredFields.orEmpty().mapNotNull { node -> + node.isAccessible = true + node.name.takeIf { node.get(nodes) is AudioProcessor && node.name in deferredNames } + } + } + + private fun stretcher(engine: AudioEngine): TimeStretcher { + val field = engine.javaClass.declaredFields.first { it.name == "stretcher" || it.name == "stretcher\$delegate" || it.name == "stretcherState" } + field.isAccessible = true + val value = field.get(engine) + return (if (value is Lazy<*>) value.value else value) as TimeStretcher + } + + private fun process(engine: AudioEngine, input: ShortArray): ShortArray { + val method = engine.javaClass.getDeclaredMethod("pushThroughDsp", ShortArray::class.java, Int::class.javaPrimitiveType) + method.isAccessible = true + method.invoke(engine, input, input.size) + val output = ShortArray(input.size * 8) + return output.copyOf(stretcher(engine).drain(output, output.size)) + } + + @Test fun idlePreferencesAndCloseDoNotBuildTheDspGraph() { + val engine = AudioEngine(isolated = true) + try { + engine.applyEffects(AudioEffectsState(isMonoEnabled = true, speed = 1.25f)) + engine.applyEqualizer(EqualizerState(isEnabled = true)) + engine.dspStages() + assertTrue(initialized(engine).isEmpty(), "Idle engine allocated: ${initialized(engine)}") + engine.stop() + assertTrue(initialized(engine).isEmpty(), "Stopping an idle engine allocated DSP") + } finally { engine.release() } + assertTrue(initialized(engine).isEmpty(), "Releasing an idle engine allocated DSP") + } + + @Test fun savedMonoReachesTheFirstPcmBlock() { + val engine = AudioEngine(isolated = true) + try { + engine.applyEffects(AudioEffectsState(isMonoEnabled = true)) + val input = shortArrayOf(1000, -2000, 3000, 1000, -8000, 4000) + assertContentEquals(shortArrayOf(-500, -500, 2000, 2000, -2000, -2000), process(engine, input)) + } finally { engine.release() } + } + + @Test fun latestSettingsApplyBeforeAudioAndBetweenBlocks() { + val engine = AudioEngine(isolated = true) + try { + val input = shortArrayOf(1000, -2000, 3000, 1000) + engine.applyEffects(AudioEffectsState(isMonoEnabled = true)) + engine.applyEffects(AudioEffectsState()) + assertContentEquals(input, process(engine, input)) + engine.applyEffects(AudioEffectsState(isMonoEnabled = true)) + assertContentEquals(shortArrayOf(-500, -500, 2000, 2000), process(engine, input)) + engine.applyEffects(AudioEffectsState()) + assertContentEquals(input, process(engine, input)) + } finally { engine.release() } + } + + @Test fun savedEqualizerReachesTheFirstPcmBlock() { + val engine = AudioEngine(isolated = true) + val reference = com.alananasss.kittytune.audio.EqualizerAudioProcessor() + try { + val state = EqualizerState(isEnabled = true, preampDb = -6f, bandGainsDb = List(16) { if (it < 4) 3f else 0f }) + val input = ShortArray(512) { ((it * 41 % 12000) - 6000).toShort() } + engine.applyEqualizer(state) + reference.configure(com.alananasss.kittytune.audio.AudioFormat(44100, 2)) + reference.setParameters(true, state.preampDb, state.bandGainsDb.toFloatArray()) + val buffer = java.nio.ByteBuffer.allocateDirect(input.size * 2).order(java.nio.ByteOrder.LITTLE_ENDIAN) + buffer.asShortBuffer().put(input) + reference.queueInput(buffer) + val output = reference.getOutput().asShortBuffer() + val expected = ShortArray(output.remaining()).also { output.get(it) } + assertContentEquals(expected, process(engine, input)) + } finally { engine.release(); reference.reset() } + } + + @Test fun releasedIdleEngineCannotCreateAnAudioGraph() { + val engine = AudioEngine(isolated = true) + engine.release() + val failure = assertFailsWith { + process(engine, shortArrayOf(1000, -1000)) + } + assertTrue(failure.cause is InterruptedException) + assertTrue(initialized(engine).isEmpty()) + } + + @Test fun automixRatioSurvivesDeferredGraphInitialization() { + val engine = AudioEngine(isolated = true) + try { + val input = ShortArray(8192) { ((it * 97 % 16000) - 8000).toShort() } + engine.applyEffects(AudioEffectsState()) + engine.setStretcherRatio(1.2f, 0.8f) + val reference = TimeStretcher(44100, 2) + reference.setParameters(1.2f, 0.8f) + reference.queue(input, input.size) + val expected = ShortArray(input.size * 8) + val count = reference.drain(expected, expected.size) + assertContentEquals(expected.copyOf(count), process(engine, input)) + } finally { engine.release() } + } +} From f69e7a73ddc31817b4118af65c147d7db80f0935 Mon Sep 17 00:00:00 2001 From: ilgiova237 Date: Wed, 18 Nov 2026 20:07:57 +0100 Subject: [PATCH 2/3] test(audio): record paired startup measurements --- experiments/deferred-dsp/EngineProbe.java | 26 ++++++ experiments/deferred-dsp/RESULTS.md | 22 +++++ experiments/deferred-dsp/startup-samples.json | 84 +++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 experiments/deferred-dsp/EngineProbe.java create mode 100644 experiments/deferred-dsp/RESULTS.md create mode 100644 experiments/deferred-dsp/startup-samples.json diff --git a/experiments/deferred-dsp/EngineProbe.java b/experiments/deferred-dsp/EngineProbe.java new file mode 100644 index 00000000..97688502 --- /dev/null +++ b/experiments/deferred-dsp/EngineProbe.java @@ -0,0 +1,26 @@ +import com.alananasss.kittytune.audio.AudioEngine; +import com.alananasss.kittytune.ui.player.AudioEffectsState; +import com.alananasss.kittytune.ui.player.EqualizerState; +import com.sun.management.ThreadMXBean; +import java.lang.management.ManagementFactory; +public class EngineProbe { + public static void main(String[] args) { + ThreadMXBean bean = (ThreadMXBean) ManagementFactory.getThreadMXBean(); + bean.setThreadAllocatedMemoryEnabled(true); + long thread = Thread.currentThread().threadId(); + for (int i=0; i<26; i++) { + long bytes = bean.getThreadAllocatedBytes(thread); + long cpu = bean.getCurrentThreadCpuTime(); + long start = System.nanoTime(); + AudioEngine engine = new AudioEngine(true); + engine.applyEffects(new AudioEffectsState()); + engine.applyEqualizer(new EqualizerState()); + engine.dspStages(); + long elapsed = System.nanoTime()-start; + cpu=bean.getCurrentThreadCpuTime()-cpu; + bytes=bean.getThreadAllocatedBytes(thread)-bytes; + System.out.println("RESULT\t"+i+"\t"+elapsed+"\t"+cpu+"\t"+bytes); + engine.release(); + } + } +} diff --git a/experiments/deferred-dsp/RESULTS.md b/experiments/deferred-dsp/RESULTS.md new file mode 100644 index 00000000..cd99704f --- /dev/null +++ b/experiments/deferred-dsp/RESULTS.md @@ -0,0 +1,22 @@ +the shared audio engine built and configured every effect processor during player construction, even before any PCM existed. saved FX/EQ settings also updated the DSP directly from the caller's thread. + +this creates the effect graph on the decoder's first PCM block and applies pending FX, EQ and stretch settings at block boundaries. it keeps the latest pre-play settings and automix ratios, configures processors before use, and releases a graph that loses a race with engine disposal. stopping or releasing an idle engine does not create DSP resources. + +this is shared code for Linux and Windows. the six new tests run in the Windows headless job and both Linux jobs. no UI, DSP equations or signal order changed. + +validation: + +- the idle-allocation regression failed on the old engine. six new tests now pass, covering idle close, first-block mono/EQ, later changes, automix ratios and use after release. +- 73 relevant local audio, limiter, trim, HLS, DRM-session and updater regressions passed. these used the installed 1.6.0 dependencies; a fresh full source build remains for CI. +- a separate review found no important lifecycle or audio-output issues. +- five alternating pairs of fresh JVM launches, with two excluded warmups, the same runtime/dependencies/profile and AOT disabled for both: median main-screen drawn + visible startup was 6.376 s before and 6.109 s after (4.2%). one pair was slower. this is a modest whole-window change, not a large whole-app speedup. + +| pair | before, ms | after, ms | +|---|---:|---:| +| 1 | 6918 | 6185 | +| 2 | 6376 | 6079 | +| 3 | 6269 | 6109 | +| 4 | 7206 | 7355 | +| 5 | 6168 | 5942 | + +filesystem caches were warm. all paired samples were retained, launches were gated on quiet CPU, and the endpoint required the actual main screen's draw marker and a visible window owned by that process. these timings are from Windows hardware; Linux coverage is functional CI, not a claimed Linux timing result. preparing the already-existing AOT cache for the personal installed build is separate work and its gain is not attributed to this PR. diff --git a/experiments/deferred-dsp/startup-samples.json b/experiments/deferred-dsp/startup-samples.json new file mode 100644 index 00000000..94d8728a --- /dev/null +++ b/experiments/deferred-dsp/startup-samples.json @@ -0,0 +1,84 @@ +[ + { + "variant": "baseline", + "run": "warmup-baseline", + "warmup": true, + "drawn_visible_ms": 7086.339 + }, + { + "variant": "candidate", + "run": "warmup-candidate", + "warmup": true, + "drawn_visible_ms": 6167.294 + }, + { + "variant": "baseline", + "run": "pair-0-baseline", + "warmup": false, + "drawn_visible_ms": 6917.769, + "pair": 0 + }, + { + "variant": "candidate", + "run": "pair-0-candidate", + "warmup": false, + "drawn_visible_ms": 6185.471, + "pair": 0 + }, + { + "variant": "candidate", + "run": "pair-1-candidate", + "warmup": false, + "drawn_visible_ms": 6079.134, + "pair": 1 + }, + { + "variant": "baseline", + "run": "pair-1-baseline", + "warmup": false, + "drawn_visible_ms": 6375.69, + "pair": 1 + }, + { + "variant": "baseline", + "run": "pair-2-baseline", + "warmup": false, + "drawn_visible_ms": 6268.94, + "pair": 2 + }, + { + "variant": "candidate", + "run": "pair-2-candidate", + "warmup": false, + "drawn_visible_ms": 6109.437, + "pair": 2 + }, + { + "variant": "candidate", + "run": "pair-3-candidate", + "warmup": false, + "drawn_visible_ms": 7355.461, + "pair": 3 + }, + { + "variant": "baseline", + "run": "pair-3-baseline", + "warmup": false, + "drawn_visible_ms": 7205.841, + "pair": 3 + }, + { + "variant": "baseline", + "run": "pair-4-baseline", + "warmup": false, + "drawn_visible_ms": 6168.188, + "pair": 4 + }, + { + "variant": "candidate", + "run": "pair-4-candidate", + "warmup": false, + "drawn_visible_ms": 5942.297, + "pair": 4 + } +] \ No newline at end of file From 52f8e5894a4d8db3d02fbe802542f4e895d8f5bd Mon Sep 17 00:00:00 2001 From: alan7383 <75907296+alan7383@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:19:03 +0100 Subject: [PATCH 3/3] fix(audio): keep DSP settings consistent within each PCM block --- .../alananasss/kittytune/audio/AudioEngine.kt | 158 ++++++++++++------ src/test/kotlin/LazyAudioEngineTest.kt | 37 +++- 2 files changed, 139 insertions(+), 56 deletions(-) diff --git a/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt b/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt index e4434c1e..440e7e19 100644 --- a/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt +++ b/src/main/kotlin/com/alananasss/kittytune/audio/AudioEngine.kt @@ -112,7 +112,28 @@ class AudioEngine(private val isolated: Boolean = false) { val nightDrive = NightDriveAudioProcessor() } - private val effectNodes = lazy { EffectNodes() } + // Share the lazy initializer lock with track metadata writers so the first block sees their latest values. + private val effectStateLock = Any() + private var vocalTrackId: Long? = null + private var vocalPositionMs = 0L + private var trackLoudness: Pair? = null + private val effectNodes = lazy(effectStateLock) { + EffectNodes().apply { + vocalRemover.setTrack(vocalTrackId) + vocalBoost.setTrack(vocalTrackId) + vocalRemover.seekToPositionMs(vocalPositionMs) + vocalBoost.seekToPositionMs(vocalPositionMs) + trackLoudness?.let { (lufs, peak) -> normalization.setTrackLoudness(lufs, peak) } + } + } + + private fun seekVocalProcessors(positionMs: Long) = synchronized(effectStateLock) { + vocalPositionMs = positionMs + if (effectNodes.isInitialized()) { + vocalRemover.seekToPositionMs(positionMs) + vocalBoost.seekToPositionMs(positionMs) + } + } private val fx get() = effectNodes.value.fx private val equalizer get() = effectNodes.value.equalizer private val reverb get() = effectNodes.value.reverb @@ -218,11 +239,15 @@ class AudioEngine(private val isolated: Boolean = false) { private val limiterState = lazy { PeakLimiter(outputSampleRate, outputChannels) } private val limiter get() = limiterState.value - @Volatile - private var effects = AudioEffectsState() - private val pendingEffects = AtomicReference(effects) - private val pendingEqualizer = AtomicReference(null) - private val pendingStretch = AtomicReference?>(1f to 1f) + private data class DspSettings( + val effects: AudioEffectsState = AudioEffectsState(), + val equalizer: EqualizerState? = null, + val speed: Float = 1f, + val pitch: Float = 1f, + ) + + private val dspSettings = AtomicReference(DspSettings()) + private var appliedDspSettings: DspSettings? = null @Volatile private var volume: Float = 1f @@ -307,8 +332,14 @@ class AudioEngine(private val isolated: Boolean = false) { // mirroring the Android MusicManager resetForNewTrack + resetResult pair. aiDetection.resetForNewTrack() if (!isolated) com.alananasss.kittytune.audio.ai.AiDetectionManager.resetResult() - vocalRemover.setTrack(trackId) - vocalBoost.setTrack(trackId) + synchronized(effectStateLock) { + vocalTrackId = trackId + if (effectNodes.isInitialized()) { + vocalRemover.setTrack(trackId) + vocalBoost.setTrack(trackId) + } + seekVocalProcessors(startPositionMs.coerceAtLeast(0L)) + } if (trackId != null && !isolated) { com.alananasss.kittytune.audio.ai.karaoke.KaraokeManager.notifyTrackChanged(trackId) } @@ -320,8 +351,6 @@ class AudioEngine(private val isolated: Boolean = false) { seekRequest.set(startPositionMs.takeIf { it > 0L }?.let { SeekRequest(it) }) if (startPositionMs > 0) { aiDetection.notifySeek(startPositionMs) - vocalRemover.seekToPositionMs(startPositionMs) - vocalBoost.seekToPositionMs(startPositionMs) } Logger.e("AudioEngine", "setMediaItem called with url: $url") @@ -537,8 +566,7 @@ class AudioEngine(private val isolated: Boolean = false) { val coerced = ms.coerceAtLeast(0) seekRequest.set(SeekRequest(coerced)) aiDetection.notifySeek(coerced) - vocalRemover.seekToPositionMs(coerced) - vocalBoost.seekToPositionMs(coerced) + seekVocalProcessors(coerced) } fun setVolume(v: Float) { @@ -570,21 +598,27 @@ class AudioEngine(private val isolated: Boolean = false) { return (positionMs - queuedMs).coerceAtLeast(0L) } - fun setTrackLoudness(lufs: Float, truePeakDb: Float = 0f) { - normalization.setTrackLoudness(lufs, truePeakDb) + fun setTrackLoudness(lufs: Float, truePeakDb: Float = 0f) = synchronized(effectStateLock) { + trackLoudness = lufs to truePeakDb + if (effectNodes.isInitialized()) normalization.setTrackLoudness(lufs, truePeakDb) } /** For a loudness learnt while the track already plays: glides rather than resets. */ - fun updateTrackLoudness(lufs: Float, truePeakDb: Float = 0f) { - normalization.updateTrackLoudness(lufs, truePeakDb) + fun updateTrackLoudness(lufs: Float, truePeakDb: Float = 0f) = synchronized(effectStateLock) { + trackLoudness = lufs to truePeakDb + if (effectNodes.isInitialized()) normalization.updateTrackLoudness(lufs, truePeakDb) } - fun clearTrackLoudness() { - normalization.clearTrackLoudness() + fun clearTrackLoudness() = synchronized(effectStateLock) { + trackLoudness = null + if (effectNodes.isInitialized()) normalization.clearTrackLoudness() } - fun getIntegratedLoudness(): Float = normalization.getIntegratedLoudness() - fun getMaxTruePeakDb(): Float = normalization.getMaxTruePeakDb() + fun getIntegratedLoudness(): Float = + if (effectNodes.isInitialized()) normalization.getIntegratedLoudness() else -70f + + fun getMaxTruePeakDb(): Float = + if (effectNodes.isInitialized()) normalization.getMaxTruePeakDb() else -120f @Synchronized fun stop() { @@ -619,24 +653,30 @@ class AudioEngine(private val isolated: Boolean = false) { } fun onKaraokeStemReady(trackId: Long) { - val curId = vocalRemover.getCurrentTrackId() ?: vocalBoost.getCurrentTrackId() - if (curId == trackId || curId == null) { - vocalRemover.setTrack(trackId) - vocalBoost.setTrack(trackId) - val prefs = com.alananasss.kittytune.data.local.PlayerPreferences() - val isAiMode = prefs.karaokeEngineMode == com.alananasss.kittytune.data.local.KaraokeEngineMode.MEL_BAND_ROFORMER - vocalRemover.setKaraokeAiMode(isAiMode) - vocalBoost.setKaraokeAiMode(isAiMode) - com.alananasss.kittytune.audio.ai.karaoke.KaraokeManager.loadStemSync(trackId) - vocalRemover.seekToPositionMs(positionMs) - vocalBoost.seekToPositionMs(positionMs) + synchronized(effectStateLock) { + if (released || (vocalTrackId != null && vocalTrackId != trackId)) return + vocalTrackId = trackId + if (effectNodes.isInitialized()) { + vocalRemover.setTrack(trackId) + vocalBoost.setTrack(trackId) + val prefs = com.alananasss.kittytune.data.local.PlayerPreferences() + val isAiMode = prefs.karaokeEngineMode == com.alananasss.kittytune.data.local.KaraokeEngineMode.MEL_BAND_ROFORMER + vocalRemover.setKaraokeAiMode(isAiMode) + vocalBoost.setKaraokeAiMode(isAiMode) + } + } + com.alananasss.kittytune.audio.ai.karaoke.KaraokeManager.loadStemSync(trackId) + synchronized(effectStateLock) { + if (!released && vocalTrackId == trackId) seekVocalProcessors(positionMs) } } fun applyEffects(state: AudioEffectsState) { - effects = state - pendingEffects.set(state) - pendingStretch.set(state.speed to if (state.isPitchEnabled) state.speed else 1f) + // Callers can resubmit the same state after changing the separate karaoke preference. + val snapshot = state.copy() + dspSettings.updateAndGet { + it.copy(effects = snapshot, speed = state.speed, pitch = if (state.isPitchEnabled) state.speed else 1f) + } } private fun applyEffectsNow(state: AudioEffectsState) { @@ -762,14 +802,14 @@ class AudioEngine(private val isolated: Boolean = false) { limiter.setCeiling(peakLimiterCeilingFor(state)) } - @Volatile - private var equalizerEnabled = false /** * The engine's processing stages in signal order, each with whether it is acting on the sound right now: * the audio-path diagnostics show it live. */ fun dspStages(): List> { + val settings = dspSettings.get() + val effects = settings.effects val mixing = com.alananasss.kittytune.audio.automix.AutomixManager.isAutomixing.value val fxOn = effectToggles.any { runCatching { it.invoke(effects) == true }.getOrDefault(false) } return listOf( @@ -780,7 +820,7 @@ class AudioEngine(private val isolated: Boolean = false) { "Automix · bass duck" to mixing, "Automix · sweep filter" to mixing, "Player FX" to fxOn, - "Equalizer" to equalizerEnabled, + "Equalizer" to (settings.equalizer?.isEnabled == true), "LUFS normalization" to effects.isNormalizationEnabled, "Time stretch" to (effects.speed != 1f), "Peak limiter" to true, @@ -788,8 +828,8 @@ class AudioEngine(private val isolated: Boolean = false) { } fun applyEqualizer(state: EqualizerState) { - equalizerEnabled = state.isEnabled - pendingEqualizer.set(state.copy(bandGainsDb = state.bandGainsDb.toList())) + val snapshot = state.copy(bandGainsDb = state.bandGainsDb.toList()) + dspSettings.updateAndGet { it.copy(equalizer = snapshot) } } private fun applyEqualizerNow(state: EqualizerState) { @@ -820,11 +860,14 @@ class AudioEngine(private val isolated: Boolean = false) { } fun setStretcherRatio(tempoRatio: Float, pitchRatio: Float) { - val baseSpeed = effects.speed - val basePitch = if (effects.isPitchEnabled) baseSpeed else 1f - val adjSpeed = (baseSpeed * tempoRatio).coerceIn(0.5f, 2.0f) - val adjPitch = (basePitch * pitchRatio).coerceIn(0.5f, 2.0f) - pendingStretch.set(adjSpeed to adjPitch) + dspSettings.updateAndGet { settings -> + val baseSpeed = settings.effects.speed + val basePitch = if (settings.effects.isPitchEnabled) baseSpeed else 1f + settings.copy( + speed = (baseSpeed * tempoRatio).coerceIn(0.5f, 2.0f), + pitch = (basePitch * pitchRatio).coerceIn(0.5f, 2.0f), + ) + } } @@ -1197,8 +1240,7 @@ class AudioEngine(private val isolated: Boolean = false) { if (!isCurrentSeek()) continue decodedPositionMs = seek aiDetection.notifySeek(seek) - vocalRemover.seekToPositionMs(seek) - vocalBoost.seekToPositionMs(seek) + seekVocalProcessors(seek) if (stretcherState.isInitialized()) stretcher.flush() if (limiterState.isInitialized()) limiter.flush() initializedDsp.get()?.forEach { it.flush() } @@ -1301,14 +1343,15 @@ class AudioEngine(private val isolated: Boolean = false) { val n = decodeToFloat(buffers, frameChannels) if (n <= 0) return - configuredProcessors() + // Keep the ceiling and downstream effects on the same settings for the whole PCM block. + val processors = configuredProcessors() limiter.process(decodeBuffer, n) if (interleaveBuffer.size < n) interleaveBuffer = ShortArray(n) for (i in 0 until n) { interleaveBuffer[i] = (decodeBuffer[i] * 32767f).toInt().coerceIn(-32768, 32767).toShort() } - pushThroughDsp(interleaveBuffer, n) + pushThroughDsp(interleaveBuffer, n, processors) } /** @@ -1368,13 +1411,21 @@ class AudioEngine(private val isolated: Boolean = false) { private fun configuredProcessors(): List { if (released) throw InterruptedException("Audio engine released") val processors = chain - pendingEffects.getAndSet(null)?.let(::applyEffectsNow) - pendingEqualizer.getAndSet(null)?.let(::applyEqualizerNow) - pendingStretch.getAndSet(null)?.let { (speed, pitch) -> stretcher.setParameters(speed, pitch) } + val settings = dspSettings.get() + val previous = appliedDspSettings + if (settings !== previous) { + if (settings.effects !== previous?.effects) applyEffectsNow(settings.effects) + if (settings.equalizer !== previous?.equalizer) settings.equalizer?.let(::applyEqualizerNow) + if (settings.speed != previous?.speed || settings.pitch != previous?.pitch) { + stretcher.setParameters(settings.speed, settings.pitch) + } + appliedDspSettings = settings + } return processors } - private fun pushThroughDsp(pcm: ShortArray, length: Int = pcm.size) { + private fun pushThroughDsp(pcm: ShortArray, length: Int, processors: List) { + if (released) throw InterruptedException("Audio engine released") val requiredBytes = length * 2 var buf = if (dspInputBuffer.capacity() < requiredBytes) { dspInputBuffer = ByteBuffer.allocateDirect(requiredBytes).order(ByteOrder.LITTLE_ENDIAN) @@ -1387,7 +1438,7 @@ class AudioEngine(private val isolated: Boolean = false) { buf.position(0) buf.limit(requiredBytes) - for (p in configuredProcessors()) { + for (p in processors) { p.queueInput(buf) buf = p.getOutput() } @@ -1405,6 +1456,7 @@ class AudioEngine(private val isolated: Boolean = false) { } private fun drainStretcher(outBuf: ShortArray, localLine: SourceDataLine?) { + if (!stretcherState.isInitialized()) return while (stretcher.available() >= outputChannels) { val n = stretcher.drain(outBuf, outBuf.size) if (n <= 0) break diff --git a/src/test/kotlin/LazyAudioEngineTest.kt b/src/test/kotlin/LazyAudioEngineTest.kt index 1c38e9ca..0a8767b5 100644 --- a/src/test/kotlin/LazyAudioEngineTest.kt +++ b/src/test/kotlin/LazyAudioEngineTest.kt @@ -37,10 +37,18 @@ class LazyAudioEngineTest { return (if (value is Lazy<*>) value.value else value) as TimeStretcher } - private fun process(engine: AudioEngine, input: ShortArray): ShortArray { - val method = engine.javaClass.getDeclaredMethod("pushThroughDsp", ShortArray::class.java, Int::class.javaPrimitiveType) + private fun beginBlock(engine: AudioEngine): List<*> { + val method = engine.javaClass.getDeclaredMethod("configuredProcessors") method.isAccessible = true - method.invoke(engine, input, input.size) + return method.invoke(engine) as List<*> + } + + private fun process(engine: AudioEngine, input: ShortArray, processors: List<*> = beginBlock(engine)): ShortArray { + val method = engine.javaClass.getDeclaredMethod( + "pushThroughDsp", ShortArray::class.java, Int::class.javaPrimitiveType, List::class.java, + ) + method.isAccessible = true + method.invoke(engine, input, input.size, processors) val output = ShortArray(input.size * 8) return output.copyOf(stretcher(engine).drain(output, output.size)) } @@ -58,6 +66,29 @@ class LazyAudioEngineTest { assertTrue(initialized(engine).isEmpty(), "Releasing an idle engine allocated DSP") } + @Test fun selectingAndSeekingAnUnplayedItemDoesNotBuildTheDspGraph() { + val engine = AudioEngine(isolated = true) + try { + engine.setMediaItem("file:///unused.wav", startPositionMs = 1_000L) + engine.seekTo(2_000L) + engine.setTrackLoudness(-12f, -1f) + assertEquals(-70f, engine.getIntegratedLoudness()) + assertEquals(-120f, engine.getMaxTruePeakDb()) + assertTrue(initialized(engine).isEmpty(), "Preparing an unplayed item allocated DSP") + } finally { engine.release() } + } + + @Test fun settingsPublishedAfterBlockStartWaitForTheNextBlock() { + val engine = AudioEngine(isolated = true) + try { + val input = shortArrayOf(1000, -2000, 3000, 1000) + val processors = beginBlock(engine) + engine.applyEffects(AudioEffectsState(isMonoEnabled = true)) + assertContentEquals(input, process(engine, input, processors)) + assertContentEquals(shortArrayOf(-500, -500, 2000, 2000), process(engine, input)) + } finally { engine.release() } + } + @Test fun savedMonoReachesTheFirstPcmBlock() { val engine = AudioEngine(isolated = true) try {