From e8b92d20c62fed11e7040f584c8de89b77cfcd33 Mon Sep 17 00:00:00 2001 From: Rhainland Date: Wed, 18 Nov 2026 17:07:40 +0100 Subject: [PATCH 1/2] fix(player): resume direct play after a later cut instead of failing A progressive direct-play load is one Media3 load task for the whole file, and its retry budget ran from the task's first failure. After a cut that resumed, the next cut minutes later (a proxy reload, an idle reset during a pause) was already past the 90 s window, or past the count-derived bound left by an earlier outage, and failed on its first retry. The player then raised an error and was rebuilt, reopening the file from the start. Each outage now gets its own window and count: a failure after an attempt that loaded at least 1 MiB starts a new one. A backend that trickles a few kilobytes and stalls again stays in the same outage and still runs out of window. HLS keeps the old behaviour, because its segment retries start over. A TLS read failure after progress is now resumable too. Android 10 reports a peer that reset the connection under a read as a bare SSLException, which nothing retried, so such a cut went straight to a player error. The retry log now records every failed direct-play load, with the cause chain's exception classes (plus HTTP status or HTTP/2 reset code) and no messages, which can carry the server's address. Refs #981 --- .../common/player/MediaLoadRetryPolicy.kt | 121 +++++++++----- .../common/player/MediaLoadRetryPolicyTest.kt | 153 +++++++++++++++--- ...gressiveDirectPlayResumeIntegrationTest.kt | 63 ++++++-- ...ecarSubtitleLoadErrorHandlingPolicyTest.kt | 18 +-- 4 files changed, 269 insertions(+), 86 deletions(-) diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt index 5be3d68926..a0e8a561c8 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt @@ -17,6 +17,8 @@ import java.net.SocketTimeoutException import java.net.UnknownHostException import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicReference +import javax.net.ssl.SSLException +import javax.net.ssl.SSLHandshakeException import okhttp3.internal.http2.ConnectionShutdownException import okhttp3.internal.http2.StreamResetException @@ -40,16 +42,18 @@ internal class SiloMediaLoadErrorHandlingPolicy( private val attemptBytesTracker = MediaLoadAttemptBytesTracker() /** - * When this load task first failed, so the retry window can be measured - * rather than inferred. + * When this load task's current outage began, so the retry window can be + * measured rather than inferred. * * `mediaLoadElapsedBeforeRetryMs` reconstructs elapsed time from the retry * count alone — it counts the delays *between* attempts and nothing spent * inside them. With a 30s socket timeout per attempt the real elapsed time * runs several times the 90s window, so a dead server kept a spinner up far * longer than intended before surfacing an error. + * + * See [MediaLoadOutage] for when a direct-play failure starts a new one. */ - private val firstErrorAtByLoadTask = ConcurrentHashMap() + private val outageByLoadTask = ConcurrentHashMap() override fun getRetryDelayMsFor(loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo): Long { val invalidResponse = loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException @@ -61,68 +65,79 @@ internal class SiloMediaLoadErrorHandlingPolicy( val resumableDirectPlay = isResumableProgressiveDirectPlay( loadErrorInfo.loadEventInfo.dataSpec.uri, ) - val firstErrorAt = firstErrorAtByLoadTask.computeIfAbsent( - loadErrorInfo.loadEventInfo.loadTaskId, - ) { nowMs() } + val now = nowMs() + val outage = outageByLoadTask.compute(loadErrorInfo.loadEventInfo.loadTaskId) { _, current -> + if (current == null || (resumableDirectPlay && bytesLoaded >= MEDIA_LOAD_OUTAGE_PROGRESS_BYTES)) { + MediaLoadOutage(startedAtMs = now, failures = 1) + } else { + current.copy(failures = current.failures + 1) + } + }!! val delayMs = siloMediaLoadRetryDelayMs( responseCode = invalidResponse?.responseCode, retryAfterHeaders = invalidResponse?.retryAfterHeaders().orEmpty(), cause = loadErrorInfo.exception, - errorCount = loadErrorInfo.errorCount, + errorCount = outage.failures, isResumableProgressiveDirectPlay = resumableDirectPlay, bytesLoaded = bytesLoaded, - observedElapsedMs = (nowMs() - firstErrorAt).coerceAtLeast(0L), - ) - logProgressiveResumeDecision( - loadErrorInfo = loadErrorInfo, - resumableDirectPlay = resumableDirectPlay, - bytesLoaded = bytesLoaded, - delayMs = delayMs, + observedElapsedMs = (now - outage.startedAtMs).coerceAtLeast(0L), ) + if (resumableDirectPlay) { + logProgressiveResumeDecision( + loadErrorInfo = loadErrorInfo, + outageFailures = outage.failures, + bytesLoaded = bytesLoaded, + delayMs = delayMs, + ) + } return delayMs } override fun onLoadTaskConcluded(loadTaskId: Long) { attemptBytesTracker.onLoadTaskConcluded(loadTaskId) - firstErrorAtByLoadTask.remove(loadTaskId) + outageByLoadTask.remove(loadTaskId) super.onLoadTaskConcluded(loadTaskId) } override fun getMinimumLoadableRetryCount(dataType: Int): Int = Int.MAX_VALUE + /** + * Logs a direct-play failure with the exception classes of its cause + * chain, never their messages, which can carry the server's address. + */ private fun logProgressiveResumeDecision( loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo, - resumableDirectPlay: Boolean, + outageFailures: Int, bytesLoaded: Long, delayMs: Long, ) { - if (!resumableDirectPlay) return - val transportFailure = loadErrorInfo.exception.findResumableTransportFailure() ?: return val resumeOffset = loadErrorInfo.loadEventInfo.dataSpec.position.coerceAtLeast(0L) + bytesLoaded.coerceAtLeast(0L) - val errorDescription = buildString { - append(transportFailure.javaClass.simpleName) - transportFailure.message?.takeIf(String::isNotBlank)?.let { - append(": ") - append(it) - } - } - if (delayMs != C.TIME_UNSET) { - SiloLog.i( + val errors = generateSequence(loadErrorInfo.exception) { it.cause } + .joinToString(separator = "/", transform = Throwable::logLabel) + val details = "error=$errors bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " + + "attempt=$outageFailures" + when { + delayMs != C.TIME_UNSET -> SiloLog.i( DiagnosticsLogCategory.PLAYBACK, TAG, - "progressive direct-play retry error=$errorDescription " + - "bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " + - "attempt=${loadErrorInfo.errorCount} retry_delay_ms=$delayMs", + "progressive direct-play retry $details retry_delay_ms=$delayMs", ) - } else { - SiloLog.w( + isRetryableMediaLoadFailure( + responseCode = (loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException)?.responseCode, + cause = loadErrorInfo.exception, + isResumableProgressiveDirectPlay = true, + bytesLoaded = bytesLoaded, + ) -> SiloLog.w( + DiagnosticsLogCategory.PLAYBACK, + TAG, + "progressive direct-play retry exhausted $details", + ) + else -> SiloLog.w( DiagnosticsLogCategory.PLAYBACK, TAG, - "progressive direct-play retry exhausted error=$errorDescription " + - "bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " + - "attempt=${loadErrorInfo.errorCount}", + "progressive direct-play load not retried $details", ) } } @@ -315,6 +330,34 @@ internal fun okhttp3.OkHttpClient.withSidecarSubtitleTimeouts(): okhttp3.OkHttpC .readTimeout(SIDECAR_SUBTITLE_READ_TIMEOUT_SECONDS, java.util.concurrent.TimeUnit.SECONDS) .build() +/** + * One run of consecutive failures of a load task, with its own retry window + * and its own [failures] count (Media3's count runs from the task's start or + * its last retry that extracted samples). + * + * A progressive direct-play load is one task for the whole file, so a failure + * there after an attempt that loaded [MEDIA_LOAD_OUTAGE_PROGRESS_BYTES] starts + * a new outage (#981): a proxy reload minutes after a resumed cut gets a full + * window, while a backend that trickles a few kilobytes and stalls again does + * not. HLS segments restart from their first byte on every retry, so their + * failures stay in one outage. + */ +internal data class MediaLoadOutage( + val startedAtMs: Long, + val failures: Int, +) + +/** About 8 s of a 1 Mbps stream, well above what a stalled backend trickles. */ +internal const val MEDIA_LOAD_OUTAGE_PROGRESS_BYTES: Long = 1L shl 20 + +/** The class, plus the HTTP status or HTTP/2 reset code where there is one; never the message. */ +private fun Throwable.logLabel(): String = + when (this) { + is HttpDataSource.InvalidResponseCodeException -> "${javaClass.simpleName}($responseCode)" + is StreamResetException -> "${javaClass.simpleName}($errorCode)" + else -> javaClass.simpleName + } + internal class MediaLoadAttemptBytesTracker { private val cumulativeBytesByLoadTask = ConcurrentHashMap() @@ -416,20 +459,20 @@ private fun Throwable.isRetryablePrematureEnd( * a ProtocolException wrapped by Media3's HttpDataSourceException. Progressive * extractors can safely reopen their current byte range after real byte * progress. HTTP/2 stream and connection shutdowns are similarly resumable, - * but only for original-file progressive delivery. + * but only for original-file progressive delivery. So is a TLS read failure, + * which Android can report as a bare `SSLException` ("Read error ... + * Connection reset by peer"); a handshake failure stays fatal. */ private fun Throwable.isResumableTransportFailure(): Boolean = this is StreamResetException || this is ConnectionShutdownException || this is EOFException || + (this is SSLException && this !is SSLHandshakeException) || (this is ProtocolException && isUnexpectedEndOfStream()) private fun ProtocolException.isUnexpectedEndOfStream(): Boolean = message?.contains("unexpected end of stream", ignoreCase = true) == true -private fun Throwable?.findResumableTransportFailure(): Throwable? = - generateSequence(this) { it.cause }.firstOrNull(Throwable::isResumableTransportFailure) - private fun Int.isRetryableMediaStatus(): Boolean = this == 404 || this == 500 || diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt index 6da3721146..2c05fe307d 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt @@ -1,20 +1,90 @@ package org.siloserver.silo.common.player +import android.net.Uri +import androidx.annotation.OptIn import androidx.media3.common.C import androidx.media3.common.ParserException +import androidx.media3.common.PlaybackException +import androidx.media3.common.util.UnstableApi +import androidx.media3.datasource.DataSpec +import androidx.media3.datasource.HttpDataSource +import androidx.media3.exoplayer.source.LoadEventInfo +import androidx.media3.exoplayer.source.MediaLoadData +import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy import java.io.EOFException import java.io.IOException import java.net.ConnectException import java.net.ProtocolException import java.net.SocketException import java.net.SocketTimeoutException +import javax.net.ssl.SSLException +import javax.net.ssl.SSLHandshakeException import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue import okhttp3.internal.http2.ErrorCode import okhttp3.internal.http2.StreamResetException +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +@OptIn(UnstableApi::class) +@RunWith(RobolectricTestRunner::class) class MediaLoadRetryPolicyTest { + private val mediaUri = Uri.parse("https://silo.test/stream/original") + private var nowMs = 1_000_000L + private val policy = SiloMediaLoadErrorHandlingPolicy( + isResumableProgressiveDirectPlay = { it == mediaUri }, + nowMs = { nowMs }, + ) + + /** + * One connection reset of load task 1, which has loaded [cumulativeBytes] + * in all. The policy keeps its own count, so Media3's [errorCount] only + * matters where a test shows it is ignored. + */ + private fun failDirectPlay(cumulativeBytes: Long, errorCount: Int = 1): Long = + policy.getRetryDelayMsFor( + loadErrorInfo( + uri = mediaUri, + loadTaskId = 1L, + error = HttpDataSource.HttpDataSourceException( + SocketException("Connection reset"), + DataSpec(mediaUri), + PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED, + HttpDataSource.HttpDataSourceException.TYPE_READ, + ), + errorCount = errorCount, + bytesLoaded = cumulativeBytes, + ), + ) + + @Test + fun `a direct-play cut long after a resumed one gets a fresh window and count`() { + // #981. Media3 resets its own count only after a retry that extracted + // samples, so a load that rode out a long outage reports a high count + // when the next one starts. + repeat(13) { + assertTrue(failDirectPlay(cumulativeBytes = 4L shl 20) > 0L) + nowMs += 5_000L + } + nowMs += 11 * 60_000L + + assertEquals(1_000L, failDirectPlay(cumulativeBytes = 900L shl 20, errorCount = 14)) + nowMs += 1_000L + assertEquals(2_000L, failDirectPlay(cumulativeBytes = 900L shl 20)) + nowMs += 2_000L + assertEquals(4_000L, failDirectPlay(cumulativeBytes = 900L shl 20)) + } + + @Test + fun `a few kilobytes between direct-play failures stay in one outage`() { + assertEquals(1_000L, failDirectPlay(cumulativeBytes = 4L shl 20)) + nowMs += 45_000L + assertEquals(2_000L, failDirectPlay(cumulativeBytes = (4L shl 20) + 8_192L)) + nowMs += 45_000L + + assertEquals(C.TIME_UNSET, failDirectPlay(cumulativeBytes = (4L shl 20) + 16_384L)) + } @Test fun `attempt progress is derived from cumulative load task bytes`() { @@ -149,33 +219,51 @@ class MediaLoadRetryPolicyTest { } @Test - fun `stream reset retries only for resumable direct play after byte progress`() { - val reset = StreamResetException(ErrorCode.INTERNAL_ERROR) - - assertEquals( - 1_000L, - siloMediaLoadRetryDelayMs( - cause = reset, - errorCount = 1, - isResumableProgressiveDirectPlay = true, - bytesLoaded = 32_768L, - ), + fun `mid-body transport failures retry only for resumable direct play after byte progress`() { + val failures = listOf( + StreamResetException(ErrorCode.INTERNAL_ERROR), + // How Android 10 reports a peer that reset a TLS connection under a read (#981). + SSLException("Read error: ssl=0x7a: I/O error during system call, Connection reset by peer"), ) + for (failure in failures) { + assertEquals( + 1_000L, + siloMediaLoadRetryDelayMs( + cause = failure, + errorCount = 1, + isResumableProgressiveDirectPlay = true, + bytesLoaded = 32_768L, + ), + "$failure", + ) + assertEquals( + C.TIME_UNSET, + siloMediaLoadRetryDelayMs( + cause = failure, + errorCount = 1, + isResumableProgressiveDirectPlay = true, + bytesLoaded = 0L, + ), + "$failure", + ) + assertEquals( + C.TIME_UNSET, + siloMediaLoadRetryDelayMs( + cause = failure, + errorCount = 1, + isResumableProgressiveDirectPlay = false, + bytesLoaded = 32_768L, + ), + "$failure", + ) + } + // A certificate the device rejects stays fatal. assertEquals( C.TIME_UNSET, siloMediaLoadRetryDelayMs( - cause = reset, + cause = SSLHandshakeException("Trust anchor for certification path not found."), errorCount = 1, isResumableProgressiveDirectPlay = true, - bytesLoaded = 0L, - ), - ) - assertEquals( - C.TIME_UNSET, - siloMediaLoadRetryDelayMs( - cause = reset, - errorCount = 1, - isResumableProgressiveDirectPlay = false, bytesLoaded = 32_768L, ), ) @@ -263,3 +351,26 @@ class MediaLoadRetryPolicyTest { assertTrue(siloMediaLoadRetryDelayMs(responseCode = 503, errorCount = 2) > 0L) } } + +/** One failed load of [uri] for the policies under test; [bytesLoaded] is the task's total. */ +@OptIn(UnstableApi::class) +internal fun loadErrorInfo( + uri: Uri, + loadTaskId: Long, + error: IOException, + errorCount: Int, + bytesLoaded: Long = 0L, +) = LoadErrorHandlingPolicy.LoadErrorInfo( + LoadEventInfo( + loadTaskId, + DataSpec(uri), + uri, + emptyMap(), + /* elapsedRealtimeMs= */ 0L, + /* loadDurationMs= */ 0L, + bytesLoaded, + ), + MediaLoadData(C.DATA_TYPE_MEDIA), + error, + errorCount, +) diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt index 7350ae0348..840e92f7e2 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/ProgressiveDirectPlayResumeIntegrationTest.kt @@ -4,6 +4,7 @@ import android.content.Context import android.net.Uri import android.os.Handler import android.os.Looper +import android.os.SystemClock import androidx.annotation.OptIn import androidx.media3.common.MediaItem import androidx.media3.common.Player @@ -25,6 +26,7 @@ import java.io.ByteArrayOutputStream import java.io.IOException import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.TimeoutException +import java.util.concurrent.atomic.AtomicLong import kotlin.test.AfterTest import kotlin.test.Test import kotlin.test.assertEquals @@ -46,7 +48,7 @@ import org.siloserver.silo.network.TokenManagerImpl @OptIn(UnstableApi::class) @RunWith(RobolectricTestRunner::class) class ProgressiveDirectPlayResumeIntegrationTest { - private val fixture = seekableWavFixture() + private val fixture = wavFixture(sampleRate = 8_000, channels = 1, bitsPerSample = 8, pcmBytes = 2_000) private val players = mutableListOf() private val clients = mutableListOf() private val servers = mutableListOf() @@ -81,6 +83,36 @@ class ProgressiveDirectPlayResumeIntegrationTest { assertEquals("\"fixture-v1\"", dispatcher.requests[1].getHeader("If-Range")) } + @Test + fun `a later cut after a successful resume gets its own retry window`() { + // #981: the stream is cut, resumes, plays on for 11 minutes and is cut + // again (a proxy reload). The second cut is a new outage, not the + // tail of the first one, so it must resume rather than surface a + // player error and rebuild the player. + // Each cut lands after more than MEDIA_LOAD_OUTAGE_PROGRESS_BYTES. + val largeFixture = wavFixture(sampleRate = 48_000, channels = 2, bitsPerSample = 16, pcmBytes = 6 shl 20) + val nowMs = AtomicLong(1_000_000L) + val dispatcher = RangeFixtureDispatcher( + largeFixture, + ResumeResponse.DISCONNECT_AGAIN, + beforeDispatch = { requestNumber -> + if (requestNumber == 2) nowMs.addAndGet(11 * 60_000L) + }, + ) + val harness = prepareHarness(dispatcher, nowMs = nowMs::get) + + awaitResumeRequest(harness.player, dispatcher) + // The second cut lands while playing; it is checked below, not thrown. + TestPlayerRunHelper.play(harness.player) + .ignoringNonFatalErrors() + .untilState(Player.STATE_ENDED) + + assertNull(harness.player.playerError) + assertEquals(3, dispatcher.requests.size) + assertEquals(2, harness.loadErrors.size) + assertTrue(harness.loadErrors.none(ObservedLoadError::wasCanceled)) + } + @Test fun `zero progress unexpected end on ranged reopen retries again`() { val dispatcher = RangeFixtureDispatcher(fixture, ResumeResponse.NO_PROGRESS) @@ -180,7 +212,10 @@ class ProgressiveDirectPlayResumeIntegrationTest { return harness.loadErrors[1] } - private fun prepareHarness(dispatcher: Dispatcher): ProgressiveHarness { + private fun prepareHarness( + dispatcher: Dispatcher, + nowMs: () -> Long = SystemClock::elapsedRealtime, + ): ProgressiveHarness { val server = MockWebServer().also { it.dispatcher = dispatcher it.start() @@ -206,6 +241,7 @@ class ProgressiveDirectPlayResumeIntegrationTest { .setLoadErrorHandlingPolicy( SiloMediaLoadErrorHandlingPolicy( isResumableProgressiveDirectPlay = { it == mediaUri }, + nowMs = nowMs, ), ) val context = ApplicationProvider.getApplicationContext() @@ -257,6 +293,7 @@ class ProgressiveDirectPlayResumeIntegrationTest { private enum class ResumeResponse { COMPLETE, + DISCONNECT_AGAIN, NO_PROGRESS, ENTITY_CHANGED, RANGE_NOT_SATISFIABLE, @@ -265,11 +302,13 @@ class ProgressiveDirectPlayResumeIntegrationTest { private class RangeFixtureDispatcher( private val fixture: ByteArray, private val resumeResponse: ResumeResponse, + private val beforeDispatch: (requestNumber: Int) -> Unit = {}, ) : Dispatcher() { val requests = CopyOnWriteArrayList() override fun dispatch(request: RecordedRequest): MockResponse { requests += request + beforeDispatch(requests.size) return if (requests.size == 1) { partialResponse(start = 0, etag = "\"fixture-v1\"") .setSocketPolicy(SocketPolicy.DISCONNECT_DURING_RESPONSE_BODY) @@ -283,6 +322,12 @@ class ProgressiveDirectPlayResumeIntegrationTest { when (resumeResponse) { ResumeResponse.COMPLETE -> partialResponse(start, etag = "\"fixture-v1\"") + ResumeResponse.DISCONNECT_AGAIN -> + partialResponse(start, etag = "\"fixture-v1\"").apply { + if (requests.size == 2) { + setSocketPolicy(SocketPolicy.DISCONNECT_DURING_RESPONSE_BODY) + } + } ResumeResponse.NO_PROGRESS -> partialResponse(start, etag = "\"fixture-v1\"", body = byteArrayOf()) .setHeader("Content-Length", fixture.size - start) @@ -323,9 +368,9 @@ class ProgressiveDirectPlayResumeIntegrationTest { generateSequence(this as Throwable?) { it.cause }.filterIsInstance().firstOrNull() private companion object { - fun seekableWavFixture(): ByteArray { - val sampleRate = 8_000 - val pcm = ByteArray(sampleRate / 4) { index -> + fun wavFixture(sampleRate: Int, channels: Int, bitsPerSample: Int, pcmBytes: Int): ByteArray { + val blockAlign = channels * bitsPerSample / 8 + val pcm = ByteArray(pcmBytes) { index -> (128 + (index % 64) - 32).toByte() } return ByteArrayOutputStream(44 + pcm.size).apply { @@ -335,11 +380,11 @@ class ProgressiveDirectPlayResumeIntegrationTest { writeAscii("fmt ") writeLittleEndian(16, 4) writeLittleEndian(1, 2) - writeLittleEndian(1, 2) - writeLittleEndian(sampleRate, 4) + writeLittleEndian(channels, 2) writeLittleEndian(sampleRate, 4) - writeLittleEndian(1, 2) - writeLittleEndian(8, 2) + writeLittleEndian(sampleRate * blockAlign, 4) + writeLittleEndian(blockAlign, 2) + writeLittleEndian(bitsPerSample, 2) writeAscii("data") writeLittleEndian(pcm.size, 4) write(pcm) diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/SidecarSubtitleLoadErrorHandlingPolicyTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/SidecarSubtitleLoadErrorHandlingPolicyTest.kt index 005ed16da9..cf8f74d594 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/SidecarSubtitleLoadErrorHandlingPolicyTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/SidecarSubtitleLoadErrorHandlingPolicyTest.kt @@ -7,9 +7,6 @@ import androidx.media3.common.PlaybackException import androidx.media3.common.util.UnstableApi import androidx.media3.datasource.DataSpec import androidx.media3.datasource.HttpDataSource -import androidx.media3.exoplayer.source.LoadEventInfo -import androidx.media3.exoplayer.source.MediaLoadData -import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy import java.io.EOFException import java.io.IOException import java.net.ProtocolException @@ -169,20 +166,7 @@ class SidecarSubtitleLoadErrorHandlingPolicyTest { } private fun errorInfo(loadTaskId: Long, error: IOException, errorCount: Int, bytesLoaded: Long = 0L) = - LoadErrorHandlingPolicy.LoadErrorInfo( - LoadEventInfo( - loadTaskId, - DataSpec(Uri.parse(URL)), - Uri.parse(URL), - emptyMap(), - /* elapsedRealtimeMs= */ 0L, - /* loadDurationMs= */ 0L, - bytesLoaded, - ), - MediaLoadData(C.DATA_TYPE_MEDIA), - error, - errorCount, - ) + loadErrorInfo(Uri.parse(URL), loadTaskId, error, errorCount, bytesLoaded) private companion object { const val URL = "https://silo.example/api/v2/stream/session/subtitles/3.vtt" From d47d2747148eed8904b3e269dda8f6d9710a2331 Mon Sep 17 00:00:00 2001 From: Rhainland Date: Wed, 18 Nov 2026 18:17:26 +0100 Subject: [PATCH 2/2] fix(player): start a new direct-play outage only after a real recovery A failure started a new retry window after any attempt that loaded 1 MiB. A load that keeps dying at the same byte re-reads from the extractor's position on each retry, so it could load that much every time and retry forever. Start a new outage only when the attempt ran at least 20 s past the previous failure, loaded at least 512 KiB, and was cut somewhere else than where the outage began. The lower threshold also lets low-bitrate streams recover. Refs #981 --- .../common/player/MediaLoadRetryPolicy.kt | 42 +++++++++---- .../common/player/MediaLoadRetryPolicyTest.kt | 60 ++++++++++++++----- 2 files changed, 77 insertions(+), 25 deletions(-) diff --git a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt index a0e8a561c8..4bad150a7d 100644 --- a/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt +++ b/android-shared/src/androidMain/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicy.kt @@ -66,11 +66,13 @@ internal class SiloMediaLoadErrorHandlingPolicy( loadErrorInfo.loadEventInfo.dataSpec.uri, ) val now = nowMs() + val cutAt = loadErrorInfo.loadEventInfo.dataSpec.position.coerceAtLeast(0L) + + bytesLoaded.coerceAtLeast(0L) val outage = outageByLoadTask.compute(loadErrorInfo.loadEventInfo.loadTaskId) { _, current -> - if (current == null || (resumableDirectPlay && bytesLoaded >= MEDIA_LOAD_OUTAGE_PROGRESS_BYTES)) { - MediaLoadOutage(startedAtMs = now, failures = 1) + if (current == null || (resumableDirectPlay && current.recoveredBefore(now, bytesLoaded, cutAt))) { + MediaLoadOutage(startedAtMs = now, failures = 1, lastFailureAtMs = now, cutAt = cutAt) } else { - current.copy(failures = current.failures + 1) + current.copy(failures = current.failures + 1, lastFailureAtMs = now) } }!! val delayMs = siloMediaLoadRetryDelayMs( @@ -336,19 +338,37 @@ internal fun okhttp3.OkHttpClient.withSidecarSubtitleTimeouts(): okhttp3.OkHttpC * its last retry that extracted samples). * * A progressive direct-play load is one task for the whole file, so a failure - * there after an attempt that loaded [MEDIA_LOAD_OUTAGE_PROGRESS_BYTES] starts - * a new outage (#981): a proxy reload minutes after a resumed cut gets a full - * window, while a backend that trickles a few kilobytes and stalls again does - * not. HLS segments restart from their first byte on every retry, so their - * failures stay in one outage. + * there starts a new outage once the load has recovered (#981): a proxy + * reload minutes after a resumed cut gets a full window. A backend that + * trickles a few kilobytes, a load that is cut again within seconds, or one + * that keeps dying at the same byte (a retry re-reads from the extractor's + * position, so it can load plenty and still stop there) stays in its outage + * and runs out of window. HLS segments restart from their first byte on + * every retry, so their failures always stay in one outage. */ internal data class MediaLoadOutage( val startedAtMs: Long, val failures: Int, -) + val lastFailureAtMs: Long, + /** Where the cut that started this outage ended, as a byte offset into the file. */ + val cutAt: Long, +) { + /** + * Whether the attempt that loaded [bytesLoaded] and was cut at [cutAt], + * at [nowMs], had recovered: it ran [MEDIA_LOAD_OUTAGE_MIN_RECOVERY_MS] + * past the previous failure, loaded [MEDIA_LOAD_OUTAGE_PROGRESS_BYTES] + * and got past where this outage began. + */ + fun recoveredBefore(nowMs: Long, bytesLoaded: Long, cutAt: Long): Boolean = + nowMs - lastFailureAtMs >= MEDIA_LOAD_OUTAGE_MIN_RECOVERY_MS && + bytesLoaded >= MEDIA_LOAD_OUTAGE_PROGRESS_BYTES && + cutAt != this.cutAt +} + +/** About 4 s of a 1 Mbps stream, or a minute of a 64 kbps audiobook; far above what a stalled backend trickles. */ +internal const val MEDIA_LOAD_OUTAGE_PROGRESS_BYTES: Long = 512L shl 10 -/** About 8 s of a 1 Mbps stream, well above what a stalled backend trickles. */ -internal const val MEDIA_LOAD_OUTAGE_PROGRESS_BYTES: Long = 1L shl 20 +internal const val MEDIA_LOAD_OUTAGE_MIN_RECOVERY_MS: Long = 20_000L /** The class, plus the HTTP status or HTTP/2 reset code where there is one; never the message. */ private fun Throwable.logLabel(): String = diff --git a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt index 2c05fe307d..dab6af1475 100644 --- a/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt +++ b/android-shared/src/androidUnitTest/kotlin/org/siloserver/silo/common/player/MediaLoadRetryPolicyTest.kt @@ -38,15 +38,22 @@ class MediaLoadRetryPolicyTest { ) /** - * One connection reset of load task 1, which has loaded [cumulativeBytes] - * in all. The policy keeps its own count, so Media3's [errorCount] only - * matters where a test shows it is ignored. + * One connection reset of an attempt that opened at [position], with + * [cumulativeBytes] loaded by the task in all. The policy keeps its own + * count, so Media3's [errorCount] only matters where a test shows it is + * ignored. */ - private fun failDirectPlay(cumulativeBytes: Long, errorCount: Int = 1): Long = + private fun failDirectPlay( + cumulativeBytes: Long, + position: Long = 0L, + errorCount: Int = 1, + loadTaskId: Long = 1L, + ): Long = policy.getRetryDelayMsFor( loadErrorInfo( uri = mediaUri, - loadTaskId = 1L, + loadTaskId = loadTaskId, + position = position, error = HttpDataSource.HttpDataSourceException( SocketException("Connection reset"), DataSpec(mediaUri), @@ -77,13 +84,34 @@ class MediaLoadRetryPolicyTest { } @Test - fun `a few kilobytes between direct-play failures stay in one outage`() { - assertEquals(1_000L, failDirectPlay(cumulativeBytes = 4L shl 20)) - nowMs += 45_000L - assertEquals(2_000L, failDirectPlay(cumulativeBytes = (4L shl 20) + 8_192L)) - nowMs += 45_000L - - assertEquals(C.TIME_UNSET, failDirectPlay(cumulativeBytes = (4L shl 20) + 16_384L)) + fun `only a meaningful recovery starts a new direct-play outage`() { + // Each case: the first cut ends at 4 MiB, then a second attempt opens + // at `position`, loads `attemptBytes` and is cut `gapMs` later. A new + // outage retries after 1 s; the same one is out of window at 89 s, or + // on its second backoff step sooner. + data class Case(val name: String, val gapMs: Long, val position: Long, val attemptBytes: Long, val expected: Long) + val mib = 1L shl 20 + val cases = listOf( + Case("recovered", gapMs = 89_000L, position = 4 * mib, attemptBytes = mib, expected = 1_000L), + Case("a low-bitrate recovery", gapMs = 89_000L, position = 4 * mib, attemptBytes = 600L shl 10, expected = 1_000L), + Case("a few kilobytes", gapMs = 89_000L, position = 4 * mib, attemptBytes = 8_192L, expected = C.TIME_UNSET), + Case("cut at the same byte", gapMs = 89_000L, position = mib, attemptBytes = 3 * mib, expected = C.TIME_UNSET), + Case("cut again within 20 s", gapMs = 10_000L, position = 4 * mib, attemptBytes = mib, expected = 2_000L), + ) + cases.forEachIndexed { index, case -> + val task = 100L + index + assertEquals(1_000L, failDirectPlay(cumulativeBytes = 4 * mib, loadTaskId = task), case.name) + nowMs += case.gapMs + assertEquals( + case.expected, + failDirectPlay( + cumulativeBytes = 4 * mib + case.attemptBytes, + position = case.position, + loadTaskId = task, + ), + case.name, + ) + } } @Test @@ -352,7 +380,10 @@ class MediaLoadRetryPolicyTest { } } -/** One failed load of [uri] for the policies under test; [bytesLoaded] is the task's total. */ +/** + * One failed load of [uri] for the policies under test; [bytesLoaded] is the + * task's total and [position] where the failed attempt opened. + */ @OptIn(UnstableApi::class) internal fun loadErrorInfo( uri: Uri, @@ -360,10 +391,11 @@ internal fun loadErrorInfo( error: IOException, errorCount: Int, bytesLoaded: Long = 0L, + position: Long = 0L, ) = LoadErrorHandlingPolicy.LoadErrorInfo( LoadEventInfo( loadTaskId, - DataSpec(uri), + DataSpec.Builder().setUri(uri).setPosition(position).build(), uri, emptyMap(), /* elapsedRealtimeMs= */ 0L,