Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import java.net.SocketTimeoutException
import java.net.UnknownHostException
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicReference
import javax.net.ssl.SSLException
import javax.net.ssl.SSLHandshakeException
import okhttp3.internal.http2.ConnectionShutdownException
import okhttp3.internal.http2.StreamResetException

Expand All @@ -40,16 +42,18 @@ internal class SiloMediaLoadErrorHandlingPolicy(
private val attemptBytesTracker = MediaLoadAttemptBytesTracker()

/**
* When this load task first failed, so the retry window can be measured
* rather than inferred.
* When this load task's current outage began, so the retry window can be
* measured rather than inferred.
*
* `mediaLoadElapsedBeforeRetryMs` reconstructs elapsed time from the retry
* count alone — it counts the delays *between* attempts and nothing spent
* inside them. With a 30s socket timeout per attempt the real elapsed time
* runs several times the 90s window, so a dead server kept a spinner up far
* longer than intended before surfacing an error.
*
* See [MediaLoadOutage] for when a direct-play failure starts a new one.
*/
private val firstErrorAtByLoadTask = ConcurrentHashMap<Long, Long>()
private val outageByLoadTask = ConcurrentHashMap<Long, MediaLoadOutage>()

override fun getRetryDelayMsFor(loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo): Long {
val invalidResponse = loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException
Expand All @@ -61,68 +65,81 @@ internal class SiloMediaLoadErrorHandlingPolicy(
val resumableDirectPlay = isResumableProgressiveDirectPlay(
loadErrorInfo.loadEventInfo.dataSpec.uri,
)
val firstErrorAt = firstErrorAtByLoadTask.computeIfAbsent(
loadErrorInfo.loadEventInfo.loadTaskId,
) { nowMs() }
val now = nowMs()
val cutAt = loadErrorInfo.loadEventInfo.dataSpec.position.coerceAtLeast(0L) +
bytesLoaded.coerceAtLeast(0L)
val outage = outageByLoadTask.compute(loadErrorInfo.loadEventInfo.loadTaskId) { _, current ->
if (current == null || (resumableDirectPlay && current.recoveredBefore(now, bytesLoaded, cutAt))) {
MediaLoadOutage(startedAtMs = now, failures = 1, lastFailureAtMs = now, cutAt = cutAt)
} else {
current.copy(failures = current.failures + 1, lastFailureAtMs = now)
}
}!!
val delayMs = siloMediaLoadRetryDelayMs(
responseCode = invalidResponse?.responseCode,
retryAfterHeaders = invalidResponse?.retryAfterHeaders().orEmpty(),
cause = loadErrorInfo.exception,
errorCount = loadErrorInfo.errorCount,
errorCount = outage.failures,
isResumableProgressiveDirectPlay = resumableDirectPlay,
bytesLoaded = bytesLoaded,
observedElapsedMs = (nowMs() - firstErrorAt).coerceAtLeast(0L),
)
logProgressiveResumeDecision(
loadErrorInfo = loadErrorInfo,
resumableDirectPlay = resumableDirectPlay,
bytesLoaded = bytesLoaded,
delayMs = delayMs,
observedElapsedMs = (now - outage.startedAtMs).coerceAtLeast(0L),
)
if (resumableDirectPlay) {
logProgressiveResumeDecision(
loadErrorInfo = loadErrorInfo,
outageFailures = outage.failures,
bytesLoaded = bytesLoaded,
delayMs = delayMs,
)
}
return delayMs
}

override fun onLoadTaskConcluded(loadTaskId: Long) {
attemptBytesTracker.onLoadTaskConcluded(loadTaskId)
firstErrorAtByLoadTask.remove(loadTaskId)
outageByLoadTask.remove(loadTaskId)
super.onLoadTaskConcluded(loadTaskId)
}

override fun getMinimumLoadableRetryCount(dataType: Int): Int =
Int.MAX_VALUE

/**
* Logs a direct-play failure with the exception classes of its cause
* chain, never their messages, which can carry the server's address.
*/
private fun logProgressiveResumeDecision(
loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo,
resumableDirectPlay: Boolean,
outageFailures: Int,
bytesLoaded: Long,
delayMs: Long,
) {
if (!resumableDirectPlay) return
val transportFailure = loadErrorInfo.exception.findResumableTransportFailure() ?: return
val resumeOffset = loadErrorInfo.loadEventInfo.dataSpec.position.coerceAtLeast(0L) +
bytesLoaded.coerceAtLeast(0L)
val errorDescription = buildString {
append(transportFailure.javaClass.simpleName)
transportFailure.message?.takeIf(String::isNotBlank)?.let {
append(": ")
append(it)
}
}
if (delayMs != C.TIME_UNSET) {
SiloLog.i(
val errors = generateSequence<Throwable>(loadErrorInfo.exception) { it.cause }
.joinToString(separator = "/", transform = Throwable::logLabel)
val details = "error=$errors bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " +
"attempt=$outageFailures"
when {
delayMs != C.TIME_UNSET -> SiloLog.i(
DiagnosticsLogCategory.PLAYBACK,
TAG,
"progressive direct-play retry error=$errorDescription " +
"bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " +
"attempt=${loadErrorInfo.errorCount} retry_delay_ms=$delayMs",
"progressive direct-play retry $details retry_delay_ms=$delayMs",
)
} else {
SiloLog.w(
isRetryableMediaLoadFailure(
responseCode = (loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException)?.responseCode,
cause = loadErrorInfo.exception,
isResumableProgressiveDirectPlay = true,
bytesLoaded = bytesLoaded,
) -> SiloLog.w(
DiagnosticsLogCategory.PLAYBACK,
TAG,
"progressive direct-play retry exhausted error=$errorDescription " +
"bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " +
"attempt=${loadErrorInfo.errorCount}",
"progressive direct-play retry exhausted $details",
)
else -> SiloLog.w(
DiagnosticsLogCategory.PLAYBACK,
TAG,
"progressive direct-play load not retried $details",
)
}
}
Expand Down Expand Up @@ -315,6 +332,52 @@ internal fun okhttp3.OkHttpClient.withSidecarSubtitleTimeouts(): okhttp3.OkHttpC
.readTimeout(SIDECAR_SUBTITLE_READ_TIMEOUT_SECONDS, java.util.concurrent.TimeUnit.SECONDS)
.build()

/**
* One run of consecutive failures of a load task, with its own retry window
* and its own [failures] count (Media3's count runs from the task's start or
* its last retry that extracted samples).
*
* A progressive direct-play load is one task for the whole file, so a failure
* there starts a new outage once the load has recovered (#981): a proxy
* reload minutes after a resumed cut gets a full window. A backend that
* trickles a few kilobytes, a load that is cut again within seconds, or one
* that keeps dying at the same byte (a retry re-reads from the extractor's
* position, so it can load plenty and still stop there) stays in its outage
* and runs out of window. HLS segments restart from their first byte on
* every retry, so their failures always stay in one outage.
*/
internal data class MediaLoadOutage(
val startedAtMs: Long,
val failures: Int,
val lastFailureAtMs: Long,
/** Where the cut that started this outage ended, as a byte offset into the file. */
val cutAt: Long,
) {
/**
* Whether the attempt that loaded [bytesLoaded] and was cut at [cutAt],
* at [nowMs], had recovered: it ran [MEDIA_LOAD_OUTAGE_MIN_RECOVERY_MS]
* past the previous failure, loaded [MEDIA_LOAD_OUTAGE_PROGRESS_BYTES]
* and got past where this outage began.
*/
fun recoveredBefore(nowMs: Long, bytesLoaded: Long, cutAt: Long): Boolean =
nowMs - lastFailureAtMs >= MEDIA_LOAD_OUTAGE_MIN_RECOVERY_MS &&
bytesLoaded >= MEDIA_LOAD_OUTAGE_PROGRESS_BYTES &&
cutAt != this.cutAt
}

/** About 4 s of a 1 Mbps stream, or a minute of a 64 kbps audiobook; far above what a stalled backend trickles. */
internal const val MEDIA_LOAD_OUTAGE_PROGRESS_BYTES: Long = 512L shl 10

internal const val MEDIA_LOAD_OUTAGE_MIN_RECOVERY_MS: Long = 20_000L

/** The class, plus the HTTP status or HTTP/2 reset code where there is one; never the message. */
private fun Throwable.logLabel(): String =
when (this) {
is HttpDataSource.InvalidResponseCodeException -> "${javaClass.simpleName}($responseCode)"
is StreamResetException -> "${javaClass.simpleName}($errorCode)"
else -> javaClass.simpleName
}

internal class MediaLoadAttemptBytesTracker {
private val cumulativeBytesByLoadTask = ConcurrentHashMap<Long, Long>()

Expand Down Expand Up @@ -416,20 +479,20 @@ private fun Throwable.isRetryablePrematureEnd(
* a ProtocolException wrapped by Media3's HttpDataSourceException. Progressive
* extractors can safely reopen their current byte range after real byte
* progress. HTTP/2 stream and connection shutdowns are similarly resumable,
* but only for original-file progressive delivery.
* but only for original-file progressive delivery. So is a TLS read failure,
* which Android can report as a bare `SSLException` ("Read error ...
* Connection reset by peer"); a handshake failure stays fatal.
*/
private fun Throwable.isResumableTransportFailure(): Boolean =
this is StreamResetException ||
this is ConnectionShutdownException ||
this is EOFException ||
(this is SSLException && this !is SSLHandshakeException) ||
(this is ProtocolException && isUnexpectedEndOfStream())

private fun ProtocolException.isUnexpectedEndOfStream(): Boolean =
message?.contains("unexpected end of stream", ignoreCase = true) == true

private fun Throwable?.findResumableTransportFailure(): Throwable? =
generateSequence(this) { it.cause }.firstOrNull(Throwable::isResumableTransportFailure)

private fun Int.isRetryableMediaStatus(): Boolean =
this == 404 ||
this == 500 ||
Expand Down
Loading
Loading