From 24f4f9a4e514ce6ba8802d2c89eae354a69f1d04 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:37:27 -0500 Subject: [PATCH 1/3] fix(player): renew direct play retries after sustained progress --- .../diagnostics/DiagnosticsInstrumentation.kt | 28 ++++++- .../common/player/MediaLoadRetryPolicy.kt | 75 ++++++++++++------- .../player/PlaybackAnalyticsListener.kt | 4 +- .../DiagnosticsInstrumentationTest.kt | 22 ++++++ .../player/DirectPlayRetryWindowTest.kt | 65 ++++++++++++++++ ...gressiveDirectPlayResumeIntegrationTest.kt | 43 ++++++----- 6 files changed, 184 insertions(+), 53 deletions(-) create mode 100644 android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentation.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentation.kt index 9eeacaace2..d9b7853dab 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentation.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentation.kt @@ -1,5 +1,9 @@ package org.siloserver.silo.common.diagnostics +import androidx.media3.common.PlaybackException +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSourceException +import java.io.IOException import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference import java.util.Locale @@ -178,9 +182,29 @@ object DiagnosticsPlaybackLogger { fun audioUnderrun() = SiloLog.w(DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "audio underrun") - fun playerError() = SiloLog.e(DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "player error") + fun playerError(error: PlaybackException) = SiloLog.e( + DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "player error ${failureDetails(error, error.errorCodeName)}", + ) + + @UnstableApi + fun loadError(error: IOException) = SiloLog.w( + DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "media load error ${failureDetails(error)}", + ) - fun loadError() = SiloLog.w(DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "media load error") + // Keep class names and Media3 codes in the message so hosted reports retain them. + // Exception messages may contain stream URLs or request headers. + @androidx.annotation.OptIn(UnstableApi::class) + private fun failureDetails(error: Throwable, playerCode: String? = null): String { + val causes = generateSequence(error) { it.cause }.take(8).toList() + val code = playerCode ?: causes.filterIsInstance().firstOrNull()?.let { + PlaybackException.getErrorCodeName(it.reason) + } + return buildString { + if (code != null) append("code=$code ") + append("causes=") + causes.joinTo(this, separator = " > ") { it.javaClass.name } + } + } // Seek and state lines carry their meaning in the message, not attributes: // the hosted collector strips position_ms and reason. Reason codes come from diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt index 5be3d68926..95449fe1a9 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt @@ -17,6 +17,7 @@ import java.net.SocketTimeoutException import java.net.UnknownHostException import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference +import kotlin.math.abs import okhttp3.internal.http2.ConnectionShutdownException import okhttp3.internal.http2.StreamResetException @@ -39,44 +40,58 @@ internal class SiloMediaLoadErrorHandlingPolicy( ) : DefaultLoadErrorHandlingPolicy() { private val attemptBytesTracker = MediaLoadAttemptBytesTracker() - /** - * When this load task first failed, so the retry window can be measured - * rather than inferred. - * - * `mediaLoadElapsedBeforeRetryMs` reconstructs elapsed time from the retry - * count alone — it counts the delays *between* attempts and nothing spent - * inside them. With a 30s socket timeout per attempt the real elapsed time - * runs several times the 90s window, so a dead server kept a spinner up far - * longer than intended before surfacing an error. - */ - private val firstErrorAtByLoadTask = ConcurrentHashMap() + private data class RetryWindow( + val startedAtMs: Long, + val lastFailureAtMs: Long, + val lastSubstantialEndOffset: Long?, + val failures: Int, + ) + + private val windowsByLoadTask = ConcurrentHashMap() override fun getRetryDelayMsFor(loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo): Long { val invalidResponse = loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException - val cumulativeBytesLoaded = loadErrorInfo.loadEventInfo.bytesLoaded - val bytesLoaded = attemptBytesTracker.bytesLoadedForAttempt( - loadTaskId = loadErrorInfo.loadEventInfo.loadTaskId, - cumulativeBytesLoaded = cumulativeBytesLoaded, - ) - val resumableDirectPlay = isResumableProgressiveDirectPlay( - loadErrorInfo.loadEventInfo.dataSpec.uri, - ) - val firstErrorAt = firstErrorAtByLoadTask.computeIfAbsent( - loadErrorInfo.loadEventInfo.loadTaskId, - ) { nowMs() } + val event = loadErrorInfo.loadEventInfo + val bytesLoaded = attemptBytesTracker.bytesLoadedForAttempt(event.loadTaskId, event.bytesLoaded) + val resumableDirectPlay = isResumableProgressiveDirectPlay(event.dataSpec.uri) + val resumeOffset = event.dataSpec.position + bytesLoaded + val now = nowMs() + val window = checkNotNull(windowsByLoadTask.compute(event.loadTaskId) { _, previous -> + // A progressive file can stay in the same load task for an entire movie. + // Repeated short reads or re-reading the same part of the file is one outage. + val substantialRead = resumableDirectPlay && bytesLoaded >= RECOVERY_BYTES + val recovered = previous != null && substantialRead && + now - previous.lastFailureAtMs >= RECOVERY_INTERVAL_MS && + (previous.lastSubstantialEndOffset == null || + abs(resumeOffset - previous.lastSubstantialEndOffset) >= RECOVERY_BYTES) + val end = if (substantialRead) resumeOffset else previous?.lastSubstantialEndOffset + if (previous == null || recovered) { + RetryWindow(now, now, end, failures = 1) + } else { + previous.copy(lastFailureAtMs = now, lastSubstantialEndOffset = end, failures = previous.failures + 1) + } + }) + // Media3 can reset its count after extraction, or carry it across a recovery. + val attempt = if (resumableDirectPlay) { + minOf(loadErrorInfo.errorCount, window.failures) + } else { + loadErrorInfo.errorCount + } val delayMs = siloMediaLoadRetryDelayMs( responseCode = invalidResponse?.responseCode, retryAfterHeaders = invalidResponse?.retryAfterHeaders().orEmpty(), cause = loadErrorInfo.exception, - errorCount = loadErrorInfo.errorCount, + errorCount = attempt, isResumableProgressiveDirectPlay = resumableDirectPlay, bytesLoaded = bytesLoaded, - observedElapsedMs = (nowMs() - firstErrorAt).coerceAtLeast(0L), + observedElapsedMs = (now - window.startedAtMs).coerceAtLeast(0L), ) logProgressiveResumeDecision( loadErrorInfo = loadErrorInfo, resumableDirectPlay = resumableDirectPlay, bytesLoaded = bytesLoaded, + resumeOffset = resumeOffset, + attempt = attempt, delayMs = delayMs, ) return delayMs @@ -84,7 +99,7 @@ internal class SiloMediaLoadErrorHandlingPolicy( override fun onLoadTaskConcluded(loadTaskId: Long) { attemptBytesTracker.onLoadTaskConcluded(loadTaskId) - firstErrorAtByLoadTask.remove(loadTaskId) + windowsByLoadTask.remove(loadTaskId) super.onLoadTaskConcluded(loadTaskId) } @@ -95,12 +110,12 @@ internal class SiloMediaLoadErrorHandlingPolicy( loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo, resumableDirectPlay: Boolean, bytesLoaded: Long, + resumeOffset: Long, + attempt: Int, delayMs: Long, ) { if (!resumableDirectPlay) return val transportFailure = loadErrorInfo.exception.findResumableTransportFailure() ?: return - val resumeOffset = loadErrorInfo.loadEventInfo.dataSpec.position.coerceAtLeast(0L) + - bytesLoaded.coerceAtLeast(0L) val errorDescription = buildString { append(transportFailure.javaClass.simpleName) transportFailure.message?.takeIf(String::isNotBlank)?.let { @@ -114,7 +129,7 @@ internal class SiloMediaLoadErrorHandlingPolicy( TAG, "progressive direct-play retry error=$errorDescription " + "bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " + - "attempt=${loadErrorInfo.errorCount} retry_delay_ms=$delayMs", + "attempt=$attempt retry_delay_ms=$delayMs", ) } else { SiloLog.w( @@ -122,13 +137,15 @@ internal class SiloMediaLoadErrorHandlingPolicy( TAG, "progressive direct-play retry exhausted error=$errorDescription " + "bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " + - "attempt=${loadErrorInfo.errorCount}", + "attempt=$attempt", ) } } private companion object { const val TAG = "MediaLoadRetry" + const val RECOVERY_BYTES = 512L * 1024L + const val RECOVERY_INTERVAL_MS = 20_000L } } diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/PlaybackAnalyticsListener.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/PlaybackAnalyticsListener.kt index 6485e69fd4..3bcb1ace0d 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/PlaybackAnalyticsListener.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/PlaybackAnalyticsListener.kt @@ -226,7 +226,7 @@ class PlaybackAnalyticsListener : AnalyticsListener { error: PlaybackException, ) { Log.e(TAG, "Player error ${error.errorCodeName}: ${error.message}", error) - DiagnosticsPlaybackLogger.playerError() + DiagnosticsPlaybackLogger.playerError(error) emit(Event.PlayerError(error)) } @@ -237,7 +237,7 @@ class PlaybackAnalyticsListener : AnalyticsListener { error: java.io.IOException, wasCanceled: Boolean, ) { - DiagnosticsPlaybackLogger.loadError() + DiagnosticsPlaybackLogger.loadError(error) emit(Event.LoadError(error)) } diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentationTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentationTest.kt index 2b8d9a9344..a1dceea25a 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentationTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/diagnostics/DiagnosticsInstrumentationTest.kt @@ -1,5 +1,9 @@ package org.siloserver.silo.common.diagnostics +import androidx.media3.common.PlaybackException +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSourceException +import javax.net.ssl.SSLException import kotlin.test.AfterTest import kotlin.test.Test import kotlin.test.assertFalse @@ -16,6 +20,24 @@ class DiagnosticsInstrumentationTest { SiloLog.installSink(null) } + @Test + @androidx.annotation.OptIn(UnstableApi::class) + fun playbackErrorsIncludeCodeAndCauseWithoutExceptionMessages() { + val lines = mutableListOf() + SiloLog.installSink { lines += it } + val code = PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED + val loadError = DataSourceException(SSLException("private stream address"), code) + DiagnosticsPlaybackLogger.loadError(loadError) + DiagnosticsPlaybackLogger.playerError(PlaybackException("private media title", loadError, code)) + + assertEquals(2, lines.size) + for (line in lines) { + assertTrue(line.contains("ERROR_CODE_IO_NETWORK_CONNECTION_FAILED"), line) + assertTrue(line.contains("javax.net.ssl.SSLException"), line) + assertFalse(line.contains("private"), line) + } + } + @Test fun networkEvidenceHasNoQueryBodyOrIdentifier() { val lines = mutableListOf() diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt new file mode 100644 index 0000000000..388fb93461 --- /dev/null +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt @@ -0,0 +1,65 @@ +package org.siloserver.silo.common.player + +import android.net.Uri +import androidx.annotation.OptIn +import androidx.media3.common.C +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSpec +import androidx.media3.exoplayer.source.LoadEventInfo +import androidx.media3.exoplayer.source.MediaLoadData +import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy +import java.net.SocketException +import kotlin.test.Test +import kotlin.test.assertEquals +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner + +@OptIn(UnstableApi::class) +@RunWith(RobolectricTestRunner::class) +class DirectPlayRetryWindowTest { + private val original = Uri.parse("https://silo.test/original") + private var now = 0L + private val policy = SiloMediaLoadErrorHandlingPolicy({ it == original }, { now }) + + private fun fail(atMs: Long, position: Long, totalBytes: Long, count: Int, uri: Uri = original): Long { + now = atMs + val dataSpec = DataSpec.Builder().setUri(uri).setPosition(position).build() + val event = LoadEventInfo.Builder(1L, dataSpec, now).setBytesLoaded(totalBytes).build() + return policy.getRetryDelayMsFor( + LoadErrorHandlingPolicy.LoadErrorInfo(event, MediaLoadData(C.DATA_TYPE_MEDIA), SocketException(), count), + ) + } + + @Test + fun laterOutageGetsFreshWindowAfterSustainedProgress() { + assertEquals(1_000L, fail(0, 0, 1_000_000, 1)) + assertEquals(1_000L, fail(660_000, 1_000_000, 1_001_000_000, 2)) + assertEquals(2_000L, fail(690_000, 1_001_000_000, 1_001_000_000, 3)) + assertEquals(C.TIME_UNSET, fail(750_000, 1_001_000_000, 1_001_000_000, 4)) + } + + @Test + fun rereadingSameBytesAfterFailedReopenDoesNotRenewWindow() { + assertEquals(1_000L, fail(0, 0, 2_000_000, 1)) + assertEquals(2_000L, fail(60_000, 0, 2_000_000, 2)) + assertEquals(C.TIME_UNSET, fail(100_000, 0, 4_000_000, 3)) + } + + @Test + fun frequentCutsExhaustWindowEvenWhenTheyDeliverData() { + for (i in 0..8) { + fail(i * 10_000L, i * 2_000_000L, (i + 1) * 2_000_000L, 2) + } + assertEquals(C.TIME_UNSET, fail(90_000, 18_000_000, 20_000_000, 2)) + } + + @Test + fun fewBytesOrLargeNonProgressiveLoadDoNotRenewWindow() { + assertEquals(1_000L, fail(0, 0, 2_000_000, 1)) + assertEquals(C.TIME_UNSET, fail(100_000, 2_000_000, 2_004_096, 2)) + policy.onLoadTaskConcluded(1L) + val segment = Uri.parse("https://silo.test/segment.ts") + assertEquals(1_000L, fail(200_000, 0, 2_000_000, 1, segment)) + assertEquals(C.TIME_UNSET, fail(300_000, 2_000_000, 8_000_000, 2, segment)) + } +} diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt index 7350ae0348..864130a261 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt @@ -62,23 +62,21 @@ class ProgressiveDirectPlayResumeIntegrationTest { } @Test - fun `progressive source resumes exact byte range and ends without player error`() { - val dispatcher = RangeFixtureDispatcher(fixture, ResumeResponse.COMPLETE) - val harness = prepareHarness(dispatcher) - - awaitResumeRequest(harness.player, dispatcher) - TestPlayerRunHelper.play(harness.player).untilState(Player.STATE_ENDED) + fun `progressive source resumes both interruptions at their byte offsets`() { + val movie = seekableWavFixture(sampleRate = 48_000, pcmBytes = 2 * 1024 * 1024) + val dispatcher = RangeFixtureDispatcher(movie, ResumeResponse.SECOND_CUT) + val harness = prepareHarness(dispatcher) { + if (dispatcher.requests.size < 2) 0L else 660_000L + } + TestPlayerRunHelper.play(harness.player).ignoringNonFatalErrors().untilState(Player.STATE_ENDED) assertNull(harness.player.playerError) - assertEquals(2, dispatcher.requests.size) - assertEquals(1, harness.loadErrors.size) - assertFalse(harness.loadErrors.single().wasCanceled) - assertNull(dispatcher.requests[0].getHeader("Range")) + assertEquals(listOf(false, false), harness.loadErrors.map { it.wasCanceled }) assertEquals( - "bytes=${fixture.size / 2}-", - dispatcher.requests[1].getHeader("Range"), + listOf(null, "bytes=${movie.size / 2}-", "bytes=${movie.size * 3 / 4}-"), + dispatcher.requests.map { it.getHeader("Range") }, ) - assertEquals("\"fixture-v1\"", dispatcher.requests[1].getHeader("If-Range")) + dispatcher.requests.drop(1).forEach { assertEquals("\"fixture-v1\"", it.getHeader("If-Range")) } } @Test @@ -180,7 +178,10 @@ class ProgressiveDirectPlayResumeIntegrationTest { return harness.loadErrors[1] } - private fun prepareHarness(dispatcher: Dispatcher): ProgressiveHarness { + private fun prepareHarness( + dispatcher: Dispatcher, + nowMs: () -> Long = android.os.SystemClock::elapsedRealtime, + ): ProgressiveHarness { val server = MockWebServer().also { it.dispatcher = dispatcher it.start() @@ -206,6 +207,7 @@ class ProgressiveDirectPlayResumeIntegrationTest { .setLoadErrorHandlingPolicy( SiloMediaLoadErrorHandlingPolicy( isResumableProgressiveDirectPlay = { it == mediaUri }, + nowMs = nowMs, ), ) val context = ApplicationProvider.getApplicationContext() @@ -256,7 +258,7 @@ class ProgressiveDirectPlayResumeIntegrationTest { ) private enum class ResumeResponse { - COMPLETE, + SECOND_CUT, NO_PROGRESS, ENTITY_CHANGED, RANGE_NOT_SATISFIABLE, @@ -281,8 +283,10 @@ class ProgressiveDirectPlayResumeIntegrationTest { ?.toInt() ?: return MockResponse().setResponseCode(400) when (resumeResponse) { - ResumeResponse.COMPLETE -> - partialResponse(start, etag = "\"fixture-v1\"") + ResumeResponse.SECOND_CUT -> + partialResponse(start, etag = "\"fixture-v1\"").apply { + if (requests.size == 2) setSocketPolicy(SocketPolicy.DISCONNECT_DURING_RESPONSE_BODY) + } ResumeResponse.NO_PROGRESS -> partialResponse(start, etag = "\"fixture-v1\"", body = byteArrayOf()) .setHeader("Content-Length", fixture.size - start) @@ -323,9 +327,8 @@ class ProgressiveDirectPlayResumeIntegrationTest { generateSequence(this as Throwable?) { it.cause }.filterIsInstance().firstOrNull() private companion object { - fun seekableWavFixture(): ByteArray { - val sampleRate = 8_000 - val pcm = ByteArray(sampleRate / 4) { index -> + fun seekableWavFixture(sampleRate: Int = 8_000, pcmBytes: Int = sampleRate / 4): ByteArray { + val pcm = ByteArray(pcmBytes) { index -> (128 + (index % 64) - 32).toByte() } return ByteArrayOutputStream(44 + pcm.size).apply { From 7f82d2703dc85d47ae2aea812184d432452ad7dd Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:49:31 -0500 Subject: [PATCH 2/3] fix(player): measure recovery between successful reads --- .../common/player/MediaLoadRetryPolicy.kt | 46 +++++++++++++-- .../silo/common/player/SiloPlayerFactory.kt | 3 +- .../player/DirectPlayRetryWindowTest.kt | 57 ++++++++++++++++--- ...gressiveDirectPlayResumeIntegrationTest.kt | 16 +++--- 4 files changed, 100 insertions(+), 22 deletions(-) diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt index 95449fe1a9..516a08a144 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt @@ -4,6 +4,8 @@ import android.net.Uri import androidx.media3.common.C import androidx.media3.common.ParserException import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSource +import androidx.media3.datasource.DataSpec import androidx.media3.datasource.HttpDataSource import androidx.media3.exoplayer.upstream.DefaultLoadErrorHandlingPolicy import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy @@ -15,6 +17,8 @@ import java.net.ProtocolException import java.net.SocketException import java.net.SocketTimeoutException import java.net.UnknownHostException +import java.util.Collections +import java.util.WeakHashMap import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference import kotlin.math.abs @@ -42,33 +46,65 @@ internal class SiloMediaLoadErrorHandlingPolicy( private data class RetryWindow( val startedAtMs: Long, - val lastFailureAtMs: Long, val lastSubstantialEndOffset: Long?, val failures: Int, ) private val windowsByLoadTask = ConcurrentHashMap() + // ProgressiveMediaPeriod passes the same DataSpec to open() and to its error + // policy. Weak keys let successful requests go away without a failure callback. + private val sustainedReads = Collections.synchronizedMap(WeakHashMap()) + + fun trackDirectPlayReads(factory: DataSource.Factory) = DataSource.Factory { + val upstream = factory.createDataSource() + object : DataSource by upstream { + private var pending: DataSpec? = null + private var firstReadMs = C.TIME_UNSET + + override fun open(dataSpec: DataSpec): Long { + pending = dataSpec.takeIf { isResumableProgressiveDirectPlay(it.uri) } + firstReadMs = C.TIME_UNSET + return upstream.open(dataSpec) + } + + override fun read(buffer: ByteArray, offset: Int, length: Int): Int { + val count = upstream.read(buffer, offset, length) + val spec = pending + if (count > 0 && spec != null) { + val now = nowMs() + if (firstReadMs == C.TIME_UNSET) { + firstReadMs = now + } else if (now - firstReadMs >= RECOVERY_INTERVAL_MS) { + sustainedReads[spec] = true + pending = null + } + } + return count + } + } + } + override fun getRetryDelayMsFor(loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo): Long { val invalidResponse = loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException val event = loadErrorInfo.loadEventInfo val bytesLoaded = attemptBytesTracker.bytesLoadedForAttempt(event.loadTaskId, event.bytesLoaded) val resumableDirectPlay = isResumableProgressiveDirectPlay(event.dataSpec.uri) val resumeOffset = event.dataSpec.position + bytesLoaded + val readOverRecoveryInterval = sustainedReads.remove(event.dataSpec) == true val now = nowMs() val window = checkNotNull(windowsByLoadTask.compute(event.loadTaskId) { _, previous -> // A progressive file can stay in the same load task for an entire movie. // Repeated short reads or re-reading the same part of the file is one outage. val substantialRead = resumableDirectPlay && bytesLoaded >= RECOVERY_BYTES - val recovered = previous != null && substantialRead && - now - previous.lastFailureAtMs >= RECOVERY_INTERVAL_MS && + val recovered = previous != null && substantialRead && readOverRecoveryInterval && (previous.lastSubstantialEndOffset == null || abs(resumeOffset - previous.lastSubstantialEndOffset) >= RECOVERY_BYTES) val end = if (substantialRead) resumeOffset else previous?.lastSubstantialEndOffset if (previous == null || recovered) { - RetryWindow(now, now, end, failures = 1) + RetryWindow(now, end, failures = 1) } else { - previous.copy(lastFailureAtMs = now, lastSubstantialEndOffset = end, failures = previous.failures + 1) + previous.copy(lastSubstantialEndOffset = end, failures = previous.failures + 1) } }) // Media3 can reset its count after extraction, or carry it across a recovery. diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/SiloPlayerFactory.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/SiloPlayerFactory.kt index 48c8c025c4..b5ae50fc81 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/SiloPlayerFactory.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/SiloPlayerFactory.kt @@ -356,6 +356,7 @@ class SiloPlayerFactory( val mediaLoadErrorHandlingPolicy = SiloMediaLoadErrorHandlingPolicy( isResumableProgressiveDirectPlay = ::isResumableDirectPlayUri, ) + val progressiveDataSources = mediaLoadErrorHandlingPolicy.trackDirectPlayReads(contentDataSourceFactory) fun correctedMediaSourceFactory( mode: DolbyVisionTransformMode, expectedDynamicRange: String? = null, @@ -372,7 +373,7 @@ class SiloPlayerFactory( dolbyVisionBaseLayerRoute = dolbyVisionBaseLayerRoute, ), ) - .setDataSourceFactory(contentDataSourceFactory) + .setDataSourceFactory(progressiveDataSources) .setSubtitleParserFactory(embeddedSubtitleParserFactory) .setLoadErrorHandlingPolicy(mediaLoadErrorHandlingPolicy) val hlsMediaSourceFactory = HlsMediaSource.Factory(contentDataSourceFactory) diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt index 388fb93461..b9da3cb836 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt @@ -4,11 +4,13 @@ import android.net.Uri import androidx.annotation.OptIn import androidx.media3.common.C import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSource import androidx.media3.datasource.DataSpec +import androidx.media3.datasource.PlaceholderDataSource import androidx.media3.exoplayer.source.LoadEventInfo import androidx.media3.exoplayer.source.MediaLoadData import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy -import java.net.SocketException +import java.net.SocketTimeoutException import kotlin.test.Test import kotlin.test.assertEquals import org.junit.runner.RunWith @@ -21,19 +23,43 @@ class DirectPlayRetryWindowTest { private var now = 0L private val policy = SiloMediaLoadErrorHandlingPolicy({ it == original }, { now }) - private fun fail(atMs: Long, position: Long, totalBytes: Long, count: Int, uri: Uri = original): Long { - now = atMs + private val source = policy.trackDirectPlayReads(DataSource.Factory { + object : DataSource by PlaceholderDataSource.INSTANCE { + override fun open(dataSpec: DataSpec): Long = C.LENGTH_UNSET.toLong() + override fun read(buffer: ByteArray, offset: Int, length: Int): Int = length + } + }).createDataSource() + + private fun fail( + atMs: Long, + position: Long, + totalBytes: Long, + count: Int, + uri: Uri = original, + firstReadMs: Long? = null, + lastReadMs: Long = atMs, + ): Long { val dataSpec = DataSpec.Builder().setUri(uri).setPosition(position).build() + source.open(dataSpec) + if (firstReadMs != null) { + val buffer = ByteArray(1) + now = firstReadMs + source.read(buffer, 0, 1) + now = lastReadMs + source.read(buffer, 0, 1) + } + source.close() + now = atMs val event = LoadEventInfo.Builder(1L, dataSpec, now).setBytesLoaded(totalBytes).build() return policy.getRetryDelayMsFor( - LoadErrorHandlingPolicy.LoadErrorInfo(event, MediaLoadData(C.DATA_TYPE_MEDIA), SocketException(), count), + LoadErrorHandlingPolicy.LoadErrorInfo(event, MediaLoadData(C.DATA_TYPE_MEDIA), SocketTimeoutException(), count), ) } @Test fun laterOutageGetsFreshWindowAfterSustainedProgress() { assertEquals(1_000L, fail(0, 0, 1_000_000, 1)) - assertEquals(1_000L, fail(660_000, 1_000_000, 1_001_000_000, 2)) + assertEquals(1_000L, fail(660_000, 1_000_000, 1_001_000_000, 2, firstReadMs = 1_000)) assertEquals(2_000L, fail(690_000, 1_001_000_000, 1_001_000_000, 3)) assertEquals(C.TIME_UNSET, fail(750_000, 1_001_000_000, 1_001_000_000, 4)) } @@ -42,7 +68,22 @@ class DirectPlayRetryWindowTest { fun rereadingSameBytesAfterFailedReopenDoesNotRenewWindow() { assertEquals(1_000L, fail(0, 0, 2_000_000, 1)) assertEquals(2_000L, fail(60_000, 0, 2_000_000, 2)) - assertEquals(C.TIME_UNSET, fail(100_000, 0, 4_000_000, 3)) + assertEquals(C.TIME_UNSET, fail(100_000, 0, 4_000_000, 3, firstReadMs = 61_000)) + } + + @Test + fun burstsSeparatedByReadTimeoutsExhaustTheOriginalWindow() { + for (i in 0..2) { + val burst = i * 31_000L + fail( + atMs = burst + 30_000, position = i * 1_000_000L, totalBytes = (i + 1) * 1_000_000L, + count = 2, firstReadMs = burst, lastReadMs = burst + 10, + ) + } + assertEquals( + C.TIME_UNSET, + fail(123_000, 3_000_000, 4_000_000, 2, firstReadMs = 93_000, lastReadMs = 93_010), + ) } @Test @@ -56,10 +97,10 @@ class DirectPlayRetryWindowTest { @Test fun fewBytesOrLargeNonProgressiveLoadDoNotRenewWindow() { assertEquals(1_000L, fail(0, 0, 2_000_000, 1)) - assertEquals(C.TIME_UNSET, fail(100_000, 2_000_000, 2_004_096, 2)) + assertEquals(C.TIME_UNSET, fail(100_000, 2_000_000, 2_004_096, 2, firstReadMs = 5_000)) policy.onLoadTaskConcluded(1L) val segment = Uri.parse("https://silo.test/segment.ts") assertEquals(1_000L, fail(200_000, 0, 2_000_000, 1, segment)) - assertEquals(C.TIME_UNSET, fail(300_000, 2_000_000, 8_000_000, 2, segment)) + assertEquals(C.TIME_UNSET, fail(300_000, 2_000_000, 8_000_000, 2, segment, firstReadMs = 220_000)) } } diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt index 864130a261..2eacfd6354 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt @@ -65,8 +65,9 @@ class ProgressiveDirectPlayResumeIntegrationTest { fun `progressive source resumes both interruptions at their byte offsets`() { val movie = seekableWavFixture(sampleRate = 48_000, pcmBytes = 2 * 1024 * 1024) val dispatcher = RangeFixtureDispatcher(movie, ResumeResponse.SECOND_CUT) + var resumedReads = 0 val harness = prepareHarness(dispatcher) { - if (dispatcher.requests.size < 2) 0L else 660_000L + if (dispatcher.requests.size >= 2 && resumedReads++ > 0) 660_000L else 0L } TestPlayerRunHelper.play(harness.player).ignoringNonFatalErrors().untilState(Player.STATE_ENDED) @@ -203,13 +204,12 @@ class ProgressiveDirectPlayResumeIntegrationTest { isResumableDirectPlayUri = { it == mediaUri }, ) } - val mediaSourceFactory = ProgressiveMediaSource.Factory(guardedFactory) - .setLoadErrorHandlingPolicy( - SiloMediaLoadErrorHandlingPolicy( - isResumableProgressiveDirectPlay = { it == mediaUri }, - nowMs = nowMs, - ), - ) + val policy = SiloMediaLoadErrorHandlingPolicy( + isResumableProgressiveDirectPlay = { it == mediaUri }, + nowMs = nowMs, + ) + val mediaSourceFactory = ProgressiveMediaSource.Factory(policy.trackDirectPlayReads(guardedFactory)) + .setLoadErrorHandlingPolicy(policy) val context = ApplicationProvider.getApplicationContext() val mediaSource = mediaSourceFactory.createMediaSource( MediaItem.fromUri(mediaUri), From ee3f3f5353c87610a1086541df55fcd7651fdad6 Mon Sep 17 00:00:00 2001 From: Quick104 <31828688+Quick104@users.noreply.github.com> Date: Wed, 18 Nov 2026 11:53:28 -0500 Subject: [PATCH 3/3] fix(player): track the byte offset of each open request --- .../common/player/MediaLoadRetryPolicy.kt | 42 ++++++++++++------- .../player/DirectPlayRetryWindowTest.kt | 36 ++++++++++++---- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt index 516a08a144..a9c034a341 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt @@ -52,36 +52,46 @@ internal class SiloMediaLoadErrorHandlingPolicy( private val windowsByLoadTask = ConcurrentHashMap() + private data class ReadProgress(val bytes: Long, val sustained: Boolean) + // ProgressiveMediaPeriod passes the same DataSpec to open() and to its error // policy. Weak keys let successful requests go away without a failure callback. - private val sustainedReads = Collections.synchronizedMap(WeakHashMap()) + private val readsByOpen = Collections.synchronizedMap(WeakHashMap()) fun trackDirectPlayReads(factory: DataSource.Factory) = DataSource.Factory { val upstream = factory.createDataSource() object : DataSource by upstream { - private var pending: DataSpec? = null + private var request: DataSpec? = null + private var bytes = 0L private var firstReadMs = C.TIME_UNSET + private var sustained = false override fun open(dataSpec: DataSpec): Long { - pending = dataSpec.takeIf { isResumableProgressiveDirectPlay(it.uri) } + request = dataSpec.takeIf { isResumableProgressiveDirectPlay(it.uri) } + bytes = 0 firstReadMs = C.TIME_UNSET + sustained = false return upstream.open(dataSpec) } override fun read(buffer: ByteArray, offset: Int, length: Int): Int { val count = upstream.read(buffer, offset, length) - val spec = pending - if (count > 0 && spec != null) { - val now = nowMs() - if (firstReadMs == C.TIME_UNSET) { - firstReadMs = now - } else if (now - firstReadMs >= RECOVERY_INTERVAL_MS) { - sustainedReads[spec] = true - pending = null + if (count > 0 && request != null) { + bytes += count + if (!sustained) { + val now = nowMs() + if (firstReadMs == C.TIME_UNSET) firstReadMs = now + sustained = now - firstReadMs >= RECOVERY_INTERVAL_MS } } return count } + + override fun close() { + request?.let { readsByOpen[it] = ReadProgress(bytes, sustained) } + request = null + upstream.close() + } } } @@ -90,14 +100,16 @@ internal class SiloMediaLoadErrorHandlingPolicy( val event = loadErrorInfo.loadEventInfo val bytesLoaded = attemptBytesTracker.bytesLoadedForAttempt(event.loadTaskId, event.bytesLoaded) val resumableDirectPlay = isResumableProgressiveDirectPlay(event.dataSpec.uri) - val resumeOffset = event.dataSpec.position + bytesLoaded - val readOverRecoveryInterval = sustainedReads.remove(event.dataSpec) == true + // The task total can include reads before an extractor seek. Only bytes from + // the DataSpec that failed belong to this request's end offset. + val read = readsByOpen.remove(event.dataSpec) + val resumeOffset = event.dataSpec.position + (read?.bytes ?: 0L) val now = nowMs() val window = checkNotNull(windowsByLoadTask.compute(event.loadTaskId) { _, previous -> // A progressive file can stay in the same load task for an entire movie. // Repeated short reads or re-reading the same part of the file is one outage. - val substantialRead = resumableDirectPlay && bytesLoaded >= RECOVERY_BYTES - val recovered = previous != null && substantialRead && readOverRecoveryInterval && + val substantialRead = read != null && read.bytes >= RECOVERY_BYTES + val recovered = previous != null && substantialRead && read.sustained && (previous.lastSubstantialEndOffset == null || abs(resumeOffset - previous.lastSubstantialEndOffset) >= RECOVERY_BYTES) val end = if (substantialRead) resumeOffset else previous?.lastSubstantialEndOffset diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt index b9da3cb836..99791d1b4f 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/DirectPlayRetryWindowTest.kt @@ -30,6 +30,16 @@ class DirectPlayRetryWindowTest { } }).createDataSource() + private var lastTotalBytes = 0L + private val buffer = ByteArray(64 * 1024) + + private fun readBytes(bytes: Long) { + var remaining = bytes + while (remaining > 0) { + remaining -= source.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + } + } + private fun fail( atMs: Long, position: Long, @@ -38,16 +48,21 @@ class DirectPlayRetryWindowTest { uri: Uri = original, firstReadMs: Long? = null, lastReadMs: Long = atMs, + bytesBeforeSeek: Long = 0, ): Long { val dataSpec = DataSpec.Builder().setUri(uri).setPosition(position).build() - source.open(dataSpec) - if (firstReadMs != null) { - val buffer = ByteArray(1) - now = firstReadMs - source.read(buffer, 0, 1) - now = lastReadMs - source.read(buffer, 0, 1) + now = firstReadMs ?: lastReadMs + if (bytesBeforeSeek > 0) { + source.open(DataSpec(uri)) + readBytes(bytesBeforeSeek) + source.close() } + source.open(dataSpec) + val bytesThisOpen = totalBytes - lastTotalBytes - bytesBeforeSeek + readBytes(bytesThisOpen / 2) + now = lastReadMs + readBytes(bytesThisOpen - bytesThisOpen / 2) + lastTotalBytes = totalBytes source.close() now = atMs val event = LoadEventInfo.Builder(1L, dataSpec, now).setBytesLoaded(totalBytes).build() @@ -71,6 +86,12 @@ class DirectPlayRetryWindowTest { assertEquals(C.TIME_UNSET, fail(100_000, 0, 4_000_000, 3, firstReadMs = 61_000)) } + @Test + fun extractorSeekDoesNotAddEarlierReadsToTheCutOffset() { + assertEquals(1_000L, fail(0, 10_000_000, 2_000_000, 1, bytesBeforeSeek = 1_000_000)) + assertEquals(1_000L, fail(660_000, 11_000_000, 3_000_000, 2, firstReadMs = 10_000)) + } + @Test fun burstsSeparatedByReadTimeoutsExhaustTheOriginalWindow() { for (i in 0..2) { @@ -99,6 +120,7 @@ class DirectPlayRetryWindowTest { assertEquals(1_000L, fail(0, 0, 2_000_000, 1)) assertEquals(C.TIME_UNSET, fail(100_000, 2_000_000, 2_004_096, 2, firstReadMs = 5_000)) policy.onLoadTaskConcluded(1L) + lastTotalBytes = 0L val segment = Uri.parse("https://silo.test/segment.ts") assertEquals(1_000L, fail(200_000, 0, 2_000_000, 1, segment)) assertEquals(C.TIME_UNSET, fail(300_000, 2_000_000, 8_000_000, 2, segment, firstReadMs = 220_000))