Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package org.siloserver.silo.common.diagnostics

import androidx.media3.common.PlaybackException
import androidx.media3.common.util.UnstableApi
import androidx.media3.datasource.DataSourceException
import java.io.IOException
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
import java.util.Locale
Expand Down Expand Up @@ -178,9 +182,29 @@ object DiagnosticsPlaybackLogger {

fun audioUnderrun() = SiloLog.w(DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "audio underrun")

fun playerError() = SiloLog.e(DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "player error")
fun playerError(error: PlaybackException) = SiloLog.e(
DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "player error ${failureDetails(error, error.errorCodeName)}",
)

@UnstableApi
fun loadError(error: IOException) = SiloLog.w(
DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "media load error ${failureDetails(error)}",
)

fun loadError() = SiloLog.w(DiagnosticsLogCategory.PLAYBACK, "Media3Analytics", "media load error")
// Keep class names and Media3 codes in the message so hosted reports retain them.
// Exception messages may contain stream URLs or request headers.
@androidx.annotation.OptIn(UnstableApi::class)
private fun failureDetails(error: Throwable, playerCode: String? = null): String {
val causes = generateSequence(error) { it.cause }.take(8).toList()
val code = playerCode ?: causes.filterIsInstance<DataSourceException>().firstOrNull()?.let {
PlaybackException.getErrorCodeName(it.reason)
}
return buildString {
if (code != null) append("code=$code ")
append("causes=")
causes.joinTo(this, separator = " > ") { it.javaClass.name }
}
}

// Seek and state lines carry their meaning in the message, not attributes:
// the hosted collector strips position_ms and reason. Reason codes come from
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import android.net.Uri
import androidx.media3.common.C
import androidx.media3.common.ParserException
import androidx.media3.common.util.UnstableApi
import androidx.media3.datasource.DataSource
import androidx.media3.datasource.DataSpec
import androidx.media3.datasource.HttpDataSource
import androidx.media3.exoplayer.upstream.DefaultLoadErrorHandlingPolicy
import androidx.media3.exoplayer.upstream.LoadErrorHandlingPolicy
Expand All @@ -15,8 +17,11 @@ import java.net.ProtocolException
import java.net.SocketException
import java.net.SocketTimeoutException
import java.net.UnknownHostException
import java.util.Collections
import java.util.WeakHashMap
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicReference
import kotlin.math.abs
import okhttp3.internal.http2.ConnectionShutdownException
import okhttp3.internal.http2.StreamResetException

Expand All @@ -39,52 +44,110 @@ internal class SiloMediaLoadErrorHandlingPolicy(
) : DefaultLoadErrorHandlingPolicy() {
private val attemptBytesTracker = MediaLoadAttemptBytesTracker()

/**
* When this load task first failed, so the retry window can be measured
* rather than inferred.
*
* `mediaLoadElapsedBeforeRetryMs` reconstructs elapsed time from the retry
* count alone — it counts the delays *between* attempts and nothing spent
* inside them. With a 30s socket timeout per attempt the real elapsed time
* runs several times the 90s window, so a dead server kept a spinner up far
* longer than intended before surfacing an error.
*/
private val firstErrorAtByLoadTask = ConcurrentHashMap<Long, Long>()
private data class RetryWindow(
val startedAtMs: Long,
val lastSubstantialEndOffset: Long?,
val failures: Int,
)

private val windowsByLoadTask = ConcurrentHashMap<Long, RetryWindow>()

private data class ReadProgress(val bytes: Long, val sustained: Boolean)

// ProgressiveMediaPeriod passes the same DataSpec to open() and to its error
// policy. Weak keys let successful requests go away without a failure callback.
private val readsByOpen = Collections.synchronizedMap(WeakHashMap<DataSpec, ReadProgress>())

fun trackDirectPlayReads(factory: DataSource.Factory) = DataSource.Factory {
val upstream = factory.createDataSource()
object : DataSource by upstream {
private var request: DataSpec? = null
private var bytes = 0L
private var firstReadMs = C.TIME_UNSET
private var sustained = false

override fun open(dataSpec: DataSpec): Long {
request = dataSpec.takeIf { isResumableProgressiveDirectPlay(it.uri) }
bytes = 0
firstReadMs = C.TIME_UNSET
sustained = false
return upstream.open(dataSpec)
}

override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
val count = upstream.read(buffer, offset, length)
if (count > 0 && request != null) {
bytes += count
if (!sustained) {
val now = nowMs()
if (firstReadMs == C.TIME_UNSET) firstReadMs = now
sustained = now - firstReadMs >= RECOVERY_INTERVAL_MS
}
}
return count
}

override fun close() {
request?.let { readsByOpen[it] = ReadProgress(bytes, sustained) }
request = null
upstream.close()
}
}
}

override fun getRetryDelayMsFor(loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo): Long {
val invalidResponse = loadErrorInfo.exception as? HttpDataSource.InvalidResponseCodeException
val cumulativeBytesLoaded = loadErrorInfo.loadEventInfo.bytesLoaded
val bytesLoaded = attemptBytesTracker.bytesLoadedForAttempt(
loadTaskId = loadErrorInfo.loadEventInfo.loadTaskId,
cumulativeBytesLoaded = cumulativeBytesLoaded,
)
val resumableDirectPlay = isResumableProgressiveDirectPlay(
loadErrorInfo.loadEventInfo.dataSpec.uri,
)
val firstErrorAt = firstErrorAtByLoadTask.computeIfAbsent(
loadErrorInfo.loadEventInfo.loadTaskId,
) { nowMs() }
val event = loadErrorInfo.loadEventInfo
val bytesLoaded = attemptBytesTracker.bytesLoadedForAttempt(event.loadTaskId, event.bytesLoaded)
val resumableDirectPlay = isResumableProgressiveDirectPlay(event.dataSpec.uri)
// The task total can include reads before an extractor seek. Only bytes from
// the DataSpec that failed belong to this request's end offset.
val read = readsByOpen.remove(event.dataSpec)
val resumeOffset = event.dataSpec.position + (read?.bytes ?: 0L)
val now = nowMs()
val window = checkNotNull(windowsByLoadTask.compute(event.loadTaskId) { _, previous ->
// A progressive file can stay in the same load task for an entire movie.
// Repeated short reads or re-reading the same part of the file is one outage.
val substantialRead = read != null && read.bytes >= RECOVERY_BYTES
val recovered = previous != null && substantialRead && read.sustained &&
Comment thread
Quick104 marked this conversation as resolved.
Resolved
(previous.lastSubstantialEndOffset == null ||
abs(resumeOffset - previous.lastSubstantialEndOffset) >= RECOVERY_BYTES)
val end = if (substantialRead) resumeOffset else previous?.lastSubstantialEndOffset
if (previous == null || recovered) {
RetryWindow(now, end, failures = 1)
} else {
previous.copy(lastSubstantialEndOffset = end, failures = previous.failures + 1)
}
})
// Media3 can reset its count after extraction, or carry it across a recovery.
val attempt = if (resumableDirectPlay) {
minOf(loadErrorInfo.errorCount, window.failures)
} else {
loadErrorInfo.errorCount
}
val delayMs = siloMediaLoadRetryDelayMs(
responseCode = invalidResponse?.responseCode,
retryAfterHeaders = invalidResponse?.retryAfterHeaders().orEmpty(),
cause = loadErrorInfo.exception,
errorCount = loadErrorInfo.errorCount,
errorCount = attempt,
isResumableProgressiveDirectPlay = resumableDirectPlay,
bytesLoaded = bytesLoaded,
observedElapsedMs = (nowMs() - firstErrorAt).coerceAtLeast(0L),
observedElapsedMs = (now - window.startedAtMs).coerceAtLeast(0L),
)
logProgressiveResumeDecision(
loadErrorInfo = loadErrorInfo,
resumableDirectPlay = resumableDirectPlay,
bytesLoaded = bytesLoaded,
resumeOffset = resumeOffset,
attempt = attempt,
delayMs = delayMs,
)
return delayMs
}

override fun onLoadTaskConcluded(loadTaskId: Long) {
attemptBytesTracker.onLoadTaskConcluded(loadTaskId)
firstErrorAtByLoadTask.remove(loadTaskId)
windowsByLoadTask.remove(loadTaskId)
super.onLoadTaskConcluded(loadTaskId)
}

Expand All @@ -95,12 +158,12 @@ internal class SiloMediaLoadErrorHandlingPolicy(
loadErrorInfo: LoadErrorHandlingPolicy.LoadErrorInfo,
resumableDirectPlay: Boolean,
bytesLoaded: Long,
resumeOffset: Long,
attempt: Int,
delayMs: Long,
) {
if (!resumableDirectPlay) return
val transportFailure = loadErrorInfo.exception.findResumableTransportFailure() ?: return
val resumeOffset = loadErrorInfo.loadEventInfo.dataSpec.position.coerceAtLeast(0L) +
bytesLoaded.coerceAtLeast(0L)
val errorDescription = buildString {
append(transportFailure.javaClass.simpleName)
transportFailure.message?.takeIf(String::isNotBlank)?.let {
Expand All @@ -114,21 +177,23 @@ internal class SiloMediaLoadErrorHandlingPolicy(
TAG,
"progressive direct-play retry error=$errorDescription " +
"bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " +
"attempt=${loadErrorInfo.errorCount} retry_delay_ms=$delayMs",
"attempt=$attempt retry_delay_ms=$delayMs",
)
} else {
SiloLog.w(
DiagnosticsLogCategory.PLAYBACK,
TAG,
"progressive direct-play retry exhausted error=$errorDescription " +
"bytes_loaded=$bytesLoaded resume_offset=$resumeOffset " +
"attempt=${loadErrorInfo.errorCount}",
"attempt=$attempt",
)
}
}

private companion object {
const val TAG = "MediaLoadRetry"
const val RECOVERY_BYTES = 512L * 1024L
const val RECOVERY_INTERVAL_MS = 20_000L
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ class PlaybackAnalyticsListener : AnalyticsListener {
error: PlaybackException,
) {
Log.e(TAG, "Player error ${error.errorCodeName}: ${error.message}", error)
DiagnosticsPlaybackLogger.playerError()
DiagnosticsPlaybackLogger.playerError(error)
emit(Event.PlayerError(error))
}

Expand All @@ -237,7 +237,7 @@ class PlaybackAnalyticsListener : AnalyticsListener {
error: java.io.IOException,
wasCanceled: Boolean,
) {
DiagnosticsPlaybackLogger.loadError()
DiagnosticsPlaybackLogger.loadError(error)
emit(Event.LoadError(error))
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,7 @@ class SiloPlayerFactory(
val mediaLoadErrorHandlingPolicy = SiloMediaLoadErrorHandlingPolicy(
isResumableProgressiveDirectPlay = ::isResumableDirectPlayUri,
)
val progressiveDataSources = mediaLoadErrorHandlingPolicy.trackDirectPlayReads(contentDataSourceFactory)
fun correctedMediaSourceFactory(
mode: DolbyVisionTransformMode,
expectedDynamicRange: String? = null,
Expand All @@ -372,7 +373,7 @@ class SiloPlayerFactory(
dolbyVisionBaseLayerRoute = dolbyVisionBaseLayerRoute,
),
)
.setDataSourceFactory(contentDataSourceFactory)
.setDataSourceFactory(progressiveDataSources)
.setSubtitleParserFactory(embeddedSubtitleParserFactory)
.setLoadErrorHandlingPolicy(mediaLoadErrorHandlingPolicy)
val hlsMediaSourceFactory = HlsMediaSource.Factory(contentDataSourceFactory)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package org.siloserver.silo.common.diagnostics

import androidx.media3.common.PlaybackException
import androidx.media3.common.util.UnstableApi
import androidx.media3.datasource.DataSourceException
import javax.net.ssl.SSLException
import kotlin.test.AfterTest
import kotlin.test.Test
import kotlin.test.assertFalse
Expand All @@ -16,6 +20,24 @@ class DiagnosticsInstrumentationTest {
SiloLog.installSink(null)
}

@Test
@androidx.annotation.OptIn(UnstableApi::class)
fun playbackErrorsIncludeCodeAndCauseWithoutExceptionMessages() {
val lines = mutableListOf<String>()
SiloLog.installSink { lines += it }
val code = PlaybackException.ERROR_CODE_IO_NETWORK_CONNECTION_FAILED
val loadError = DataSourceException(SSLException("private stream address"), code)
DiagnosticsPlaybackLogger.loadError(loadError)
DiagnosticsPlaybackLogger.playerError(PlaybackException("private media title", loadError, code))

assertEquals(2, lines.size)
for (line in lines) {
assertTrue(line.contains("ERROR_CODE_IO_NETWORK_CONNECTION_FAILED"), line)
assertTrue(line.contains("javax.net.ssl.SSLException"), line)
assertFalse(line.contains("private"), line)
}
}

@Test
fun networkEvidenceHasNoQueryBodyOrIdentifier() {
val lines = mutableListOf<String>()
Expand Down
Loading
Loading