diff --git a/android/AGENTS.md b/android/AGENTS.md index 233ac1c..0b3bed6 100644 --- a/android/AGENTS.md +++ b/android/AGENTS.md @@ -16,7 +16,8 @@ Repository-wide rules (languages and localization, naming) are in the root `AGEN while a key editor is visible). There is no navigation rail. `BoxWithConstraints` is used instead of `WindowSizeClass` because the size classes put 841 and 1056 dp in the same 840 dp bucket. The Wide page measures its own width: >= 900 dp gives three profile columns and zoom beside touch, otherwise two columns and stacked. - At 1691x978 dp the Wide page must not scroll (covered by `SettingsWideTest`). + At 1691x978 dp the Wide page must not scroll (covered by `SettingsWideTest`); for that the Wide page shows Video + (codec and quality) as one row that opens a dialog, like the Connection hosts. - `Sections.kt`: shared components. Colours and type come from `MaterialTheme` only. - `DisplaySection.kt`: the Display section (screensaver switch, idle / brightness / blank-after / shape speed sliders, movement mode (cross the screen or bounce), preview, replay of the getting-started guide). The Wide page must stay diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt index 8f08f23..b9aca50 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/Capabilities.kt @@ -1,11 +1,15 @@ package com.librestatic.ezdisplay import android.app.Activity +import android.content.SharedPreferences import android.media.MediaCodecInfo import android.media.MediaCodecList import android.os.Build import android.view.Display import android.view.InputDevice +import com.librestatic.ezdisplay.settings.SettingsStore +import com.librestatic.ezdisplay.settings.StreamQualities +import com.librestatic.ezdisplay.settings.VideoCodecs import org.json.JSONArray import org.json.JSONObject import kotlin.math.max @@ -39,6 +43,26 @@ object Capabilities { return NativeModes(native.first, native.second, modes) } + /** True when the device lists a decoder for [codec] (`h264`, `hevc`, `av1`). */ + fun hasDecoder(codec: String): Boolean { + val mime = CODECS.firstOrNull { it.first == codec }?.second ?: return false + return MediaCodecList(MediaCodecList.REGULAR_CODECS).codecInfos.any { info -> + !info.isEncoder && info.supportedTypes.any { it.equals(mime, ignoreCase = true) } + } + } + + /** + * `stream_prefs` from the settings. HEVC without an HEVC decoder is sent as `auto`; unknown or missing values + * are `auto` / `normal`. + */ + fun streamPrefs(prefs: SharedPreferences, hevcAvailable: Boolean = hasDecoder("hevc")): JSONObject { + val codec = prefs.getString(SettingsStore.VIDEO_CODEC, null).takeIf { it in VideoCodecs } ?: "auto" + val quality = prefs.getString(SettingsStore.STREAM_QUALITY, null).takeIf { it in StreamQualities } ?: "normal" + return JSONObject() + .put("codec", if (codec == "hevc" && !hevcAvailable) "auto" else codec) + .put("quality", quality) + } + fun toJson(activity: Activity, chosen: Display.Mode?): String { val display = display(activity) val native = nativeModes(display) @@ -59,6 +83,7 @@ object Capabilities { .put("decoders", decoders(native.width, native.height)) .put("stylus", hasStylus()) .put("max_touch_points", 10) + .put("stream_prefs", streamPrefs(InputPrefs.prefs(activity))) return json.toString() } diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt index abd4793..ef5582f 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/MainActivity.kt @@ -130,6 +130,7 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. } private var chosenMode: android.view.Display.Mode? = null private var capsJson: String? = null + private val hevcAvailable by lazy { Capabilities.hasDecoder("hevc") } private var started = false // between onStart and onStop private var surfaceReady = false @@ -160,6 +161,9 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. private var floatBuf = FloatArray(512) private var timeBuf = LongArray(64) private var statusHideAt = 0L + private lateinit var osdView: StreamOsdView + private var osdHideAt = 0L + private var osdSeq = -1 private var cursorSeq = 0L private var cursorShapeId = -1 @@ -296,6 +300,13 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. root.addView(telemetryView, FrameLayout.LayoutParams( ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT, Gravity.TOP or Gravity.START, )) + osdView = StreamOsdView(this) + root.addView(osdView, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT, Gravity.BOTTOM or Gravity.START, + ).apply { + val m = (16 * resources.displayMetrics.density).toInt() + setMargins(m, m, m, m) + }) settingsButton = ImageButton(this).apply { setImageResource(android.R.drawable.ic_menu_preferences) setBackgroundColor(0x99000000.toInt()) @@ -582,8 +593,12 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. private fun connect() { if (!started || sessionActive) return if (!surfaceReady) return // surfaceCreated reschedules + val base = capsJson ?: Capabilities.toJson(this, chosenMode).also { capsJson = it } + // Codec, quality and the audio switch may change in the settings while this activity stays alive. val caps = Capabilities.withAudio( - capsJson ?: Capabilities.toJson(this, chosenMode).also { capsJson = it }, + JSONObject(base) + .put("stream_prefs", Capabilities.streamPrefs(InputPrefs.prefs(this), hevcAvailable)) + .toString(), InputPrefs.prefs(this).getBoolean(SettingsStore.AUDIO_ENABLED, true), ) pushKeyboardState() @@ -815,6 +830,7 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. } val wifiChoice = activeWifi val usb = activeAccessory + if (j.optString("phase") != "streaming") hideOsd(false) val addr = if (wifiChoice != null) hostLabel(wifiChoice) else "$HOST:$PORT" when (j.optString("phase")) { "connecting" -> setStatus( @@ -832,6 +848,16 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. }, ) "streaming" -> { + // Shown at stream start and again whenever the host sends a new report. + val info = StreamOsd.parse(j) + if (!wasStreaming || info.seq != osdSeq) { + osdSeq = info.seq + osdView.show(StreamOsd.render(resources, info)) + osdHideAt = now + StreamOsd.SHOW_MS + } else if (osdHideAt != 0L && now >= osdHideAt) { + osdHideAt = 0 + osdView.hide(true) + } if (!wasStreaming) { wasStreaming = true pairingLost = null @@ -899,6 +925,12 @@ class MainActivity : ComponentActivity(), SurfaceHolder.Callback, Choreographer. } } + private fun hideOsd(animated: Boolean) { + osdHideAt = 0 + osdSeq = -1 + osdView.hide(animated) + } + private fun showTransient(text: String, ms: Long) { setStatus(text) statusHideAt = SystemClock.uptimeMillis() + ms diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/StreamOsd.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/StreamOsd.kt new file mode 100644 index 0000000..8a890b6 --- /dev/null +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/StreamOsd.kt @@ -0,0 +1,226 @@ +package com.librestatic.ezdisplay + +import android.content.Context +import android.content.res.Resources +import android.graphics.Color +import android.graphics.Typeface +import android.graphics.drawable.GradientDrawable +import android.text.TextUtils +import android.util.TypedValue +import android.view.View +import android.widget.LinearLayout +import android.widget.TextView +import org.json.JSONObject +import java.util.Locale + +/** One encoder the host tried to open. Encoder, options and error come from the host and are not translated. */ +data class OsdAttempt(val encoder: String, val options: String, val ok: Boolean, val error: String) + +/** The host's negotiation report (status JSON `report`). */ +data class OsdReport( + val wantedKbps: Int, + val userBitrate: Boolean, + val ceilingKbps: Int, + val attempts: List, + val rateMode: String, + val quality: Int, + val targetKbps: Int, + val maxKbps: Int, + /** `normal` / `high` / `lossless`; `normal` and `auto` when an older host does not report them. */ + val wantedQuality: String = "normal", + val appliedQuality: String = "normal", + val wantedCodec: String = "auto", +) + +/** What the on-screen display needs from one status poll. */ +data class OsdInfo( + val seq: Int, + val codec: String, + val bitrateKbps: Int, + val width: Int, + val height: Int, + val refreshMhz: Int, + val report: OsdReport?, +) + +/** Text of the display: small monospace [lines] and the larger [headline]. */ +data class OsdContent(val lines: List, val headline: String) + +object StreamOsd { + const val MAX_ATTEMPTS = 6 + const val SHOW_MS = 6000L + const val FADE_MS = 400L + private const val MAX_ERROR_CHARS = 60 + + /** Reads the fields of the status JSON. A missing `report_seq` or `report` (older native lib or host) is none. */ + fun parse(j: JSONObject): OsdInfo { + val r = j.optJSONObject("report") + val report = r?.let { + val list = it.optJSONArray("attempts") + OsdReport( + wantedKbps = it.optInt("wanted_kbps"), + userBitrate = it.optBoolean("user_bitrate"), + ceilingKbps = it.optInt("ceiling_kbps"), + attempts = (0 until (list?.length() ?: 0)).mapNotNull { i -> list?.optJSONObject(i) }.map { a -> + OsdAttempt(a.optString("encoder"), a.optString("options"), a.optBoolean("ok"), a.optString("error")) + }, + rateMode = it.optString("rate_mode", "unknown"), + quality = it.optInt("quality"), + targetKbps = it.optInt("target_kbps"), + maxKbps = it.optInt("max_kbps"), + wantedQuality = it.optString("wanted_quality", "normal"), + appliedQuality = it.optString("applied_quality", "normal"), + wantedCodec = it.optString("wanted_codec", "auto"), + ) + } + return OsdInfo( + seq = j.optInt("report_seq"), + codec = j.optString("codec"), + bitrateKbps = j.optInt("bitrate_kbps"), + width = j.optInt("width"), + height = j.optInt("height"), + refreshMhz = j.optInt("refresh_mhz"), + report = report, + ) + } + + fun codecName(codec: String): String = when (codec.lowercase(Locale.ROOT)) { + "h264" -> "H.264" + "hevc" -> "HEVC" + "av1" -> "AV1" + else -> codec.uppercase(Locale.ROOT) + } + + /** "46 Mbit/s": one decimal below 10, none from 10 up. */ + fun mbit(res: Resources, kbps: Int): String = res.getString(R.string.osd_mbit, mbitNumber(res, kbps)) + + private fun locale(res: Resources): Locale = res.configuration.locales[0] + + private fun mbitNumber(res: Resources, kbps: Int): String { + val mbit = kbps / 1000.0 + return if (mbit < 10) String.format(locale(res), "%.1f", mbit) else String.format(locale(res), "%d", Math.round(mbit)) + } + + private fun qualityName(res: Resources, q: String): String = when (q) { + "high" -> res.getString(R.string.osd_quality_high) + "lossless" -> res.getString(R.string.osd_quality_lossless) + else -> res.getString(R.string.osd_quality_normal) + } + + private fun hz(res: Resources, mhz: Int): String = + if (mhz % 1000 == 0) (mhz / 1000).toString() else String.format(locale(res), "%.2f", mhz / 1000.0) + + private fun ellipsize(s: String): String = + if (s.length <= MAX_ERROR_CHARS) s else s.take(MAX_ERROR_CHARS - 1).trimEnd() + "…" + + fun render(res: Resources, info: OsdInfo): OsdContent { + val report = info.report + val lines = mutableListOf() + if (report != null) { + val parts = mutableListOf(res.getString(R.string.osd_negotiated, mbit(res, info.bitrateKbps))) + parts += if (report.userBitrate) { + res.getString(R.string.osd_set_in_settings) + } else { + res.getString(R.string.osd_auto_for, "${info.width}×${info.height}", hz(res, info.refreshMhz)) + } + if (report.wantedKbps > report.ceilingKbps && report.ceilingKbps > 0) { + parts += res.getString( + R.string.osd_clamped, mbitNumber(res, report.wantedKbps), mbitNumber(res, report.ceilingKbps), + ) + } + if (report.ceilingKbps > 0) parts += res.getString(R.string.osd_link_limit, mbit(res, report.ceilingKbps)) + lines += parts.joinToString(" · ") + if (report.wantedQuality != "normal") { + lines += if (report.appliedQuality == report.wantedQuality) { + res.getString(R.string.osd_quality, qualityName(res, report.wantedQuality)) + } else { + res.getString( + R.string.osd_quality_applied, + qualityName(res, report.wantedQuality), qualityName(res, report.appliedQuality), + ) + } + } + if (report.wantedCodec != "auto" && !report.wantedCodec.equals(info.codec, ignoreCase = true)) { + lines += res.getString(R.string.osd_codec_wanted, codecName(report.wantedCodec)) + } + val hidden = (report.attempts.size - MAX_ATTEMPTS).coerceAtLeast(0) + if (hidden > 0) lines += res.getQuantityString(R.plurals.osd_more_attempts, hidden, hidden) + for (a in report.attempts.drop(hidden)) { + val head = "${if (a.ok) "✓" else "✗"} ${a.encoder}" + if (a.options.isNotEmpty()) " ${a.options}" else "" + lines += if (!a.ok && a.error.isNotEmpty()) "$head — ${ellipsize(a.error.lineSequence().first())}" else head + } + } + val headline = mutableListOf(codecName(info.codec)) + if (report != null) { + when (report.rateMode) { + "cbr" -> res.getString(R.string.osd_rate_cbr) + "vbr" -> res.getString(R.string.osd_rate_vbr, report.quality) + "cqp" -> res.getString(R.string.osd_rate_cqp, report.quality) + "crf" -> res.getString(R.string.osd_rate_crf, report.quality) + "lossless" -> res.getString(R.string.osd_rate_lossless) + else -> null + }?.let { headline += it } + } + // A fixed quantiser or lossless stream has no target rate: the negotiated one would mislead. + val fixedQuality = report?.rateMode == "cqp" || report?.rateMode == "lossless" + if (!fixedQuality) { + headline += if (report != null && report.maxKbps > info.bitrateKbps && info.bitrateKbps > 0) { + res.getString(R.string.osd_rate_peak, mbitNumber(res, info.bitrateKbps), mbitNumber(res, report.maxKbps)) + } else { + mbit(res, info.bitrateKbps) + } + } + return OsdContent(lines, headline.joinToString(" · ")) + } +} + +/** Small, non-interactive overlay for the bottom-left corner. */ +class StreamOsdView(context: Context) : LinearLayout(context) { + private val detail = TextView(context).apply { + setTextColor(Color.WHITE) + setTextSize(TypedValue.COMPLEX_UNIT_SP, 11f) + typeface = Typeface.MONOSPACE + maxLines = 10 + ellipsize = TextUtils.TruncateAt.END + } + private val headline = TextView(context).apply { + setTextColor(Color.WHITE) + setTextSize(TypedValue.COMPLEX_UNIT_SP, 19f) + typeface = Typeface.DEFAULT_BOLD + } + + init { + orientation = VERTICAL + val pad = (10 * resources.displayMetrics.density).toInt() + setPadding(pad, pad, pad, pad) + background = GradientDrawable().apply { + setColor(0x99000000.toInt()) + cornerRadius = 8 * resources.displayMetrics.density + } + isClickable = false + isFocusable = false + importantForAccessibility = IMPORTANT_FOR_ACCESSIBILITY_NO + addView(detail, LayoutParams(LayoutParams.WRAP_CONTENT, LayoutParams.WRAP_CONTENT)) + addView(headline, LayoutParams(LayoutParams.WRAP_CONTENT, LayoutParams.WRAP_CONTENT)) + visibility = GONE + } + + fun show(content: OsdContent) { + detail.text = content.lines.joinToString("\n") + detail.visibility = if (content.lines.isEmpty()) GONE else VISIBLE + headline.text = content.headline + animate().cancel() + alpha = 1f + visibility = VISIBLE + } + + fun hide(animated: Boolean) { + if (visibility != VISIBLE) return + if (animated) { + animate().alpha(0f).setDuration(StreamOsd.FADE_MS).withEndAction { visibility = GONE } + } else { + animate().cancel() + visibility = GONE + } + } +} diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt index 6827f99..c822b7c 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/Sections.kt @@ -18,14 +18,23 @@ import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.heightIn +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.TextButton +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.ui.window.Dialog +import androidx.compose.ui.window.DialogProperties import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.ColorScheme import androidx.compose.material3.DropdownMenu import androidx.compose.material3.DropdownMenuItem import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.ExperimentalMaterial3ExpressiveApi +import androidx.compose.material3.FilledTonalButton import androidx.compose.material3.ExposedDropdownMenuAnchorType import androidx.compose.material3.ExposedDropdownMenu import androidx.compose.material3.ExposedDropdownMenuBox @@ -57,6 +66,7 @@ import androidx.compose.ui.graphics.Shape import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.testTag +import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.res.painterResource import androidx.compose.ui.res.stringResource import androidx.compose.ui.semantics.Role @@ -429,6 +439,123 @@ private fun TriggerRow(t: Trigger, state: SettingsState, actions: SettingsAction } } +// ---------------------------------------------------------------- Video + +/** Pref value to the label of a video codec choice. */ +internal fun videoCodecLabel(v: String): Int = CodecLabels[v] ?: R.string.video_codec_auto + +internal fun videoQualityLabel(v: String): Int = QualityLabels[v] ?: R.string.video_quality_normal + +private val CodecLabels = mapOf( + "auto" to R.string.video_codec_auto, "h264" to R.string.video_codec_h264, "hevc" to R.string.video_codec_hevc, +) + +private val QualityLabels = mapOf( + "normal" to R.string.video_quality_normal, "high" to R.string.video_quality_high, + "lossless" to R.string.video_quality_lossless, +) + +/** Video codec and quality choices as one segmented group; they apply on the next connection. */ +@OptIn(ExperimentalMaterial3ExpressiveApi::class) +@Composable +fun VideoGroup(state: SettingsState, actions: SettingsActions, modifier: Modifier = Modifier) { + val colors = MaterialTheme.colorScheme + Column(modifier.fillMaxWidth().testTag("video_group"), verticalArrangement = Arrangement.spacedBy(2.dp)) { + VideoChoice( + tag = "video_codec", title = R.string.video_codec_title, desc = R.string.video_codec_desc, + values = VideoCodecs, labels = CodecLabels, selected = state.videoCodec, + enabled = { it != "hevc" || state.hevcAvailable }, onSelect = actions::setVideoCodec, + shape = segmentShape(0, 2), colors = colors, + ) + VideoChoice( + tag = "stream_quality", title = R.string.video_quality_title, desc = R.string.video_quality_desc, + values = StreamQualities, labels = QualityLabels, selected = state.streamQuality, + enabled = { true }, onSelect = actions::setStreamQuality, + shape = segmentShape(1, 2), colors = colors, + ) + } +} + +@OptIn(ExperimentalMaterial3ExpressiveApi::class) +@Composable +private fun VideoChoice( + tag: String, + @StringRes title: Int, + @StringRes desc: Int, + values: List, + labels: Map, + selected: String, + enabled: (String) -> Boolean, + onSelect: (String) -> Unit, + shape: Shape, + colors: ColorScheme, +) { + Surface(shape = shape, color = colors.surfaceBright, modifier = Modifier.fillMaxWidth().testTag(tag)) { + Column(Modifier.padding(RowPadding), verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text(stringResource(title), style = MaterialTheme.typography.titleSmall, fontWeight = FontWeight.Bold) + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(ButtonGroupDefaults.ConnectedSpaceBetween)) { + values.forEachIndexed { i, v -> + ToggleButton( + checked = selected == v, + onCheckedChange = { onSelect(v) }, + enabled = enabled(v), + shapes = when (i) { + 0 -> ButtonGroupDefaults.connectedLeadingButtonShapes() + values.lastIndex -> ButtonGroupDefaults.connectedTrailingButtonShapes() + else -> ButtonGroupDefaults.connectedMiddleButtonShapes() + }, + modifier = Modifier.weight(1f).semantics { role = Role.RadioButton }.testTag("${tag}_$v"), + ) { + Text(stringResource(labels.getValue(v)), maxLines = 1, overflow = TextOverflow.Ellipsis) + } + } + } + Text(stringResource(desc), style = MaterialTheme.typography.bodySmall, color = colors.onSurfaceVariant) + } + } +} + +/** + * Wide page version: a button on the title row with the current choices that opens [VideoGroup] in a dialog, so the + * page gains no height (the key editor pane leaves no room for another row). + */ +@Composable +fun VideoSummary(state: SettingsState, actions: SettingsActions, modifier: Modifier = Modifier) { + val colors = MaterialTheme.colorScheme + var open by rememberSaveable { mutableStateOf(false) } + FilledTonalButton(onClick = { open = true }, modifier = modifier.testTag("video_summary")) { + Symbol(R.drawable.ic_tune, size = 20.dp) + Text( + stringResource( + R.string.video_summary, + stringResource(videoCodecLabel(state.videoCodec)), stringResource(videoQualityLabel(state.streamQuality)), + ), + Modifier.padding(start = 8.dp), + ) + } + if (open) { + Dialog(onDismissRequest = { open = false }, properties = DialogProperties(usePlatformDefaultWidth = false)) { + Surface( + shape = RoundedCornerShape(28.dp), color = colors.surfaceContainer, + modifier = Modifier.widthIn(max = 560.dp).fillMaxWidth(0.9f).testTag("video_dialog"), + ) { + Column(Modifier.verticalScroll(rememberScrollState()).padding(24.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) { + Row(verticalAlignment = Alignment.CenterVertically) { + Text( + stringResource(R.string.section_video), Modifier.weight(1f), + style = MaterialTheme.typography.headlineSmall, fontWeight = FontWeight.Bold, + ) + TextButton(onClick = { open = false }, modifier = Modifier.testTag("video_dialog_close")) { + Text(stringResource(R.string.wifi_manage_close)) + } + } + VideoGroup(state, actions) + } + } + } + } +} + // ---------------------------------------------------------------- Zoom, palm, air actions /** Zoom level buttons and double-click time slider as one segmented group. */ diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt index 6b220c1..7ab618b 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsScreen.kt @@ -56,7 +56,7 @@ import com.librestatic.ezdisplay.InputPrefs import com.librestatic.ezdisplay.R /** Settings sections; AirActions only exists while its card is shown (and is always last). */ -enum class SettingsSection { Pen, Zoom, Touch, Audio, Connection, Display, AirActions } +enum class SettingsSection { Pen, Zoom, Touch, Audio, Video, Connection, Display, AirActions } private fun airCardShown(state: SettingsState) = state.airActions == AirActions.On || state.airActions == AirActions.Unknown @@ -181,6 +181,8 @@ private fun PhoneMain(state: SettingsState, actions: SettingsActions, onOpenEdit AirActionsCard(state.airActions, Modifier.padding(top = profileGap(2))) SectionTitle(stringResource(R.string.section_audio)) AudioRow(state.audioEnabled, actions::setAudioEnabled) + SectionTitle(stringResource(R.string.section_video)) + VideoGroup(state, actions) SectionTitle(stringResource(R.string.section_connection)) ConnectionContent(state, actions) SectionTitle(stringResource(R.string.section_display)) @@ -233,7 +235,8 @@ private fun ListDetailLayout( ) { val colors = MaterialTheme.colorScheme val sections = buildList { - add(SettingsSection.Pen); add(SettingsSection.Zoom); add(SettingsSection.Touch); add(SettingsSection.Audio); add(SettingsSection.Connection); add(SettingsSection.Display) + add(SettingsSection.Pen); add(SettingsSection.Zoom); add(SettingsSection.Touch); add(SettingsSection.Audio) + add(SettingsSection.Video); add(SettingsSection.Connection); add(SettingsSection.Display) if (airCardShown(state)) add(SettingsSection.AirActions) } val current = selected.takeIf { it in sections } ?: SettingsSection.Pen @@ -293,6 +296,7 @@ private fun ListDetailLayout( SettingsSection.Zoom -> ZoomGroup(state, actions) SettingsSection.Touch -> PalmRow(state.palmRejection, actions::setPalmRejection) SettingsSection.Audio -> AudioRow(state.audioEnabled, actions::setAudioEnabled) + SettingsSection.Video -> VideoGroup(state, actions) SettingsSection.Connection -> ConnectionContent(state, actions) SettingsSection.Display -> DisplayContent(state, actions) SettingsSection.AirActions -> AirActionsCard(state.airActions) @@ -307,6 +311,7 @@ private fun sectionTitle(s: SettingsSection) = when (s) { SettingsSection.Zoom -> R.string.section_zoom SettingsSection.Touch -> R.string.settings_palm_title SettingsSection.Audio -> R.string.section_audio + SettingsSection.Video -> R.string.section_video SettingsSection.Connection -> R.string.section_connection SettingsSection.Display -> R.string.section_display SettingsSection.AirActions -> R.string.section_air_actions @@ -317,6 +322,7 @@ private fun sectionIcon(s: SettingsSection) = when (s) { SettingsSection.Zoom -> R.drawable.ic_zoom_in SettingsSection.Touch -> R.drawable.ic_back_hand SettingsSection.Audio -> R.drawable.ic_volume_up + SettingsSection.Video -> R.drawable.ic_tune SettingsSection.Connection -> R.drawable.ic_wifi SettingsSection.Display -> R.drawable.ic_bedtime SettingsSection.AirActions -> R.drawable.ic_info @@ -328,6 +334,9 @@ private fun sectionValue(s: SettingsSection, state: SettingsState): String = whe SettingsSection.Zoom -> stringResource(R.string.settings_zoom_summary, formatZoom(state.zoomFactor), state.doubleClickMs) SettingsSection.Touch -> stringResource(if (state.palmRejection) R.string.settings_on else R.string.settings_off) SettingsSection.Audio -> stringResource(if (state.audioEnabled) R.string.settings_on else R.string.settings_off) + SettingsSection.Video -> stringResource( + R.string.video_summary, stringResource(videoCodecLabel(state.videoCodec)), stringResource(videoQualityLabel(state.streamQuality)), + ) SettingsSection.Connection -> connectionSummary(state) SettingsSection.Display -> displaySummary(state) SettingsSection.AirActions -> stringResource(R.string.section_air_actions_value) @@ -418,6 +427,7 @@ private fun WideLayout( Modifier.weight(1f), style = MaterialTheme.typography.displayMedium, fontWeight = FontWeight.ExtraBold, ) + VideoSummary(state, actions, Modifier.padding(end = 12.dp)) DisplaySheetButton(state, { displaySheet = true }) } if (displaySheet) DisplaySheet(state, actions) { displaySheet = false } diff --git a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt index 3b4974e..3476e00 100644 --- a/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt +++ b/android/app/src/main/kotlin/com/librestatic/ezdisplay/settings/SettingsStore.kt @@ -9,6 +9,7 @@ import android.provider.Settings import androidx.core.content.edit import androidx.compose.runtime.State import androidx.compose.runtime.mutableStateOf +import com.librestatic.ezdisplay.Capabilities import com.librestatic.ezdisplay.InputPrefs import com.librestatic.ezdisplay.InputPrefs.KeyCombo import com.librestatic.ezdisplay.R @@ -124,6 +125,12 @@ val KeyPresets = listOf( KeyPreset(R.string.keys_preset_ctrl_shift_z, KeyCombo(InputPrefs.MOD_CTRL or InputPrefs.MOD_SHIFT, 0x1D)), ) +/** Video codec choices (`stream_prefs.codec`) in display order. */ +val VideoCodecs = listOf("auto", "h264", "hevc") + +/** Quality choices (`stream_prefs.quality`) in display order. */ +val StreamQualities = listOf("normal", "high", "lossless") + /** Everything the settings screen shows. Defaults match what [InputPrefs.toJson] uses for unset prefs. */ data class SettingsState( val profile: String = "raw", @@ -152,6 +159,12 @@ data class SettingsState( val audioEnabled: Boolean = true, /** Connect over Wi-Fi when the USB connection is not there. */ val wifiEnabled: Boolean = false, + /** Preferred video codec: one of [VideoCodecs]. */ + val videoCodec: String = "auto", + /** Quality over USB: one of [StreamQualities]. */ + val streamQuality: String = "normal", + /** Whether this device has an HEVC decoder; the HEVC choice is disabled without one. */ + val hevcAvailable: Boolean = true, val wifiHosts: List = emptyList(), /** Hosts seen on the network right now (only while discovery runs). */ val discovered: List = emptyList(), @@ -265,6 +278,8 @@ interface SettingsActions { /** Opens the getting-started guide again. */ fun replayOnboarding() {} fun setAudioEnabled(on: Boolean) + fun setVideoCodec(codec: String) + fun setStreamQuality(quality: String) /** Opens the PIN dialog for a host that was found or typed. */ fun startPairing(name: String, shortFp: String, addr: String) @@ -284,6 +299,7 @@ class SettingsStore( airActions: AirActions, private val wifi: WifiBackend = WifiBackend.None, private val deviceName: String = Build.MODEL, + private val hevcAvailable: Boolean = true, /** Runs the blocking pairing; replaced by a direct executor in tests. */ private val io: Executor = Executors.newSingleThreadExecutor(), private val panelType: PanelType = PanelType.Unknown, @@ -320,6 +336,9 @@ class SettingsStore( airActions = airActions, audioEnabled = prefs.getBoolean(AUDIO_ENABLED, true), wifiEnabled = prefs.getBoolean(WIFI_ENABLED, false), + videoCodec = prefs.getString(VIDEO_CODEC, null).takeIf { it in VideoCodecs } ?: "auto", + streamQuality = prefs.getString(STREAM_QUALITY, null).takeIf { it in StreamQualities } ?: "normal", + hevcAvailable = hevcAvailable, wifiHosts = wifi.hosts(), deviceFingerprint = wifi.identity(), ) @@ -426,6 +445,19 @@ class SettingsStore( _state.value = _state.value.copy(wifiEnabled = on) } + /** Ignored for an unknown codec, and for HEVC on a device without an HEVC decoder. */ + override fun setVideoCodec(codec: String) { + if (codec !in VideoCodecs || (codec == "hevc" && !hevcAvailable)) return + prefs.edit { putString(VIDEO_CODEC, codec) } + _state.value = _state.value.copy(videoCodec = codec) + } + + override fun setStreamQuality(quality: String) { + if (quality !in StreamQualities) return + prefs.edit { putString(STREAM_QUALITY, quality) } + _state.value = _state.value.copy(streamQuality = quality) + } + /** Hosts seen on the network changed. */ fun setDiscovered(hosts: List) { _state.value = _state.value.copy(discovered = hosts) @@ -491,6 +523,10 @@ class SettingsStore( /** Pref key of the Wi-Fi switch, read by MainActivity. */ const val WIFI_ENABLED = "wifi_enabled" + /** Pref keys of the video choices, read by `Capabilities` (applied on the next connection). */ + const val VIDEO_CODEC = "video_codec" + const val STREAM_QUALITY = "stream_quality" + /** Pref key of the audio switch, read by MainActivity when a session starts. */ const val AUDIO_ENABLED = "audio_enabled" @@ -500,6 +536,7 @@ class SettingsStore( onReplayOnboarding: () -> Unit = {}, ) = SettingsStore( InputPrefs.prefs(context), readAirActions(context), NativeWifiBackend(context), + hevcAvailable = Capabilities.hasDecoder("hevc"), panelType = PanelDetector.cached(context), onPreviewScreensaver = onPreviewScreensaver, onReplayOnboarding = onReplayOnboarding, ) diff --git a/android/app/src/main/res/values-de/strings.xml b/android/app/src/main/res/values-de/strings.xml index d3bacbd..124e9df 100644 --- a/android/app/src/main/res/values-de/strings.xml +++ b/android/app/src/main/res/values-de/strings.xml @@ -155,6 +155,39 @@ %1$d gekoppelter Host %1$d gekoppelte Hosts + Ausgehandelt: %1$s + automatisch für %1$s bei %2$s Hz + in den Einstellungen festgelegt + gewünscht %1$s, begrenzt auf %2$s + Verbindungslimit %1$s + %1$s Mbit/s + %1$s Mbit/s (Spitze %2$s) + konstante Bitrate + variable Bitrate, Qualität %1$d + konstante Qualität %1$d + konstanter Ratenfaktor %1$d + + +%1$d weiteres + +%1$d weitere + + + Video + Video-Codec + Automatisch + Automatisch wählt den besten Codec, den dieses Tablet und der PC unterstützen. Änderungen gelten ab der nächsten Verbindung. + Qualität über USB + Normal + Hoch + Verlustfrei + Hoch ist eine feste, nahezu verlustfreie Qualität und braucht mehr Daten. Verlustfrei nutzt nahezu verlustfreie Qualität, solange sich das Bild bewegt, und zeigt das exakte (verlustfreie, 4:2:0) Bild einen Moment nach dem Stillstand: die meisten Daten. Beides gilt nur über USB; WLAN nutzt immer Normal. + normal + hoch + verlustfrei + Qualität: %1$s + Qualität: %1$s angefordert · %2$s angewendet + Codec: %1$s angefordert + verlustfrei + %1$s · %2$s Anzeige Bildschirmschoner aus diff --git a/android/app/src/main/res/values-es/strings.xml b/android/app/src/main/res/values-es/strings.xml index 5d00a72..9369093 100644 --- a/android/app/src/main/res/values-es/strings.xml +++ b/android/app/src/main/res/values-es/strings.xml @@ -157,6 +157,40 @@ %1$d hosts vinculados %1$d hosts vinculados + Negociado: %1$s + auto para %1$s a %2$s Hz + definido en ajustes + pedido %1$s, limitado a %2$s + límite del enlace %1$s + %1$s Mbit/s + %1$s Mbit/s (pico %2$s) + tasa de bits constante + tasa de bits variable, calidad %1$d + calidad constante %1$d + factor de tasa constante %1$d + + +%1$d más + +%1$d más + +%1$d más + + + Video + Códec de video + Automático + Automático elige el mejor códec que admitan tanto esta tablet como la PC. Los cambios se aplican en la próxima conexión. + Calidad por USB + Normal + Alta + Sin pérdida + Alta es una calidad fija casi sin pérdida y usa más datos. Sin pérdida usa calidad casi sin pérdida mientras la imagen se mueve y muestra la imagen exacta (sin pérdida, 4:2:0) un instante después de que se detiene: la mayor cantidad de datos. Ambas solo se aplican por USB; el Wi-Fi siempre usa Normal. + normal + alta + sin pérdida + Calidad: %1$s + Calidad: %1$s solicitada · %2$s aplicada + Códec: %1$s solicitado + sin pérdida + %1$s · %2$s Pantalla Protector desactivado diff --git a/android/app/src/main/res/values-fr/strings.xml b/android/app/src/main/res/values-fr/strings.xml index a758632..3b9cde1 100644 --- a/android/app/src/main/res/values-fr/strings.xml +++ b/android/app/src/main/res/values-fr/strings.xml @@ -157,6 +157,40 @@ %1$d hôtes associés %1$d hôtes associés + Négocié : %1$s + auto pour %1$s à %2$s Hz + défini dans les réglages + demandé %1$s, limité à %2$s + limite de la liaison %1$s + %1$s Mbit/s + %1$s Mbit/s (crête %2$s) + débit constant + débit variable, qualité %1$d + qualité constante %1$d + facteur de débit constant %1$d + + +%1$d de plus + +%1$d de plus + +%1$d de plus + + + Vidéo + Codec vidéo + Automatique + Automatique choisit le meilleur codec pris en charge par cette tablette et par le PC. Les changements s\'appliquent à la prochaine connexion. + Qualité via USB + Normale + Élevée + Sans perte + Élevée est une qualité fixe quasi sans perte et utilise plus de données. Sans perte utilise une qualité quasi sans perte tant que l\'image bouge, puis affiche l\'image exacte (sans perte, 4:2:0) un instant après son arrêt : le plus de données. Les deux ne s\'appliquent que via USB ; le Wi-Fi utilise toujours Normale. + normale + élevée + sans perte + Qualité : %1$s + Qualité : %1$s demandée · %2$s appliquée + Codec : %1$s demandé + sans perte + %1$s · %2$s Écran Économiseur désactivé diff --git a/android/app/src/main/res/values-it/strings.xml b/android/app/src/main/res/values-it/strings.xml index 7c7581f..3d0dd3b 100644 --- a/android/app/src/main/res/values-it/strings.xml +++ b/android/app/src/main/res/values-it/strings.xml @@ -157,6 +157,40 @@ %1$d host associati %1$d host associati + Negoziato: %1$s + auto per %1$s a %2$s Hz + impostato nelle impostazioni + richiesto %1$s, limitato a %2$s + limite del collegamento %1$s + %1$s Mbit/s + %1$s Mbit/s (picco %2$s) + bitrate costante + bitrate variabile, qualità %1$d + qualità costante %1$d + fattore di velocità costante %1$d + + +%1$d altro + +%1$d altri + +%1$d altri + + + Video + Codec video + Automatico + Automatico sceglie il miglior codec supportato sia da questo tablet sia dal PC. Le modifiche valgono dalla prossima connessione. + Qualità via USB + Normale + Alta + Senza perdite + Alta è una qualità fissa quasi senza perdite e usa più dati. Senza perdite usa una qualità quasi senza perdite mentre l\'immagine si muove e mostra l\'immagine esatta (senza perdite, 4:2:0) un istante dopo che si ferma: il massimo dei dati. Entrambe valgono solo via USB; il Wi-Fi usa sempre Normale. + normale + alta + senza perdite + Qualità: %1$s + Qualità: %1$s richiesta · %2$s applicata + Codec: %1$s richiesto + senza perdite + %1$s · %2$s Display Salvaschermo disattivato diff --git a/android/app/src/main/res/values-pt/strings.xml b/android/app/src/main/res/values-pt/strings.xml index 0a0da50..6863e8d 100644 --- a/android/app/src/main/res/values-pt/strings.xml +++ b/android/app/src/main/res/values-pt/strings.xml @@ -157,6 +157,40 @@ %1$d hosts pareados %1$d hosts pareados + Negociado: %1$s + automático para %1$s a %2$s Hz + definido nas configurações + desejado %1$s, limitado a %2$s + limite do link %1$s + %1$s Mbit/s + %1$s Mbit/s (pico %2$s) + taxa de bits constante + taxa de bits variável, qualidade %1$d + qualidade constante %1$d + fator de taxa constante %1$d + + +%1$d a mais + +%1$d a mais + +%1$d a mais + + + Vídeo + Codec de vídeo + Automático + Automático escolhe o melhor codec compatível com este tablet e com o PC. As alterações valem a partir da próxima conexão. + Qualidade via USB + Normal + Alta + Sem perdas + Alta é uma qualidade fixa quase sem perdas e usa mais dados. Sem perdas usa qualidade quase sem perdas enquanto a imagem se move e mostra a imagem exata (sem perdas, 4:2:0) um instante depois que ela para: o máximo de dados. Ambas só valem via USB; o Wi-Fi sempre usa Normal. + normal + alta + sem perdas + Qualidade: %1$s + Qualidade: %1$s solicitada · %2$s aplicada + Codec: %1$s solicitado + sem perdas + %1$s · %2$s Tela Protetor desativado diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml index 71d2a1c..ffa7725 100644 --- a/android/app/src/main/res/values/strings.xml +++ b/android/app/src/main/res/values/strings.xml @@ -155,6 +155,41 @@ %1$d paired host %1$d paired hosts + Negotiated %1$s + auto for %1$s at %2$s Hz + set in settings + wanted %1$s, limited to %2$s + link limit %1$s + %1$s Mbit/s + %1$s Mbit/s (peak %2$s) + constant bitrate + variable bitrate, quality %1$d + constant quality %1$d + constant rate factor %1$d + + +%1$d more + +%1$d more + + + Video + Video codec + Automatic + H.264 + HEVC + Automatic picks the best codec that both this tablet and the PC support. Changes apply on the next connection. + Quality over USB + Normal + High + Lossless + High is near-lossless fixed quality and uses more data. Lossless uses near-lossless quality while the image moves and shows the exact image (lossless 4:2:0) a moment after it stops: the most data. Both only apply over USB; Wi-Fi always uses Normal. + normal + high + lossless + Quality: %1$s + Quality: %1$s requested · %2$s applied + Codec: %1$s requested + lossless + %1$s · %2$s Display Screensaver off diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/StreamOsdTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/StreamOsdTest.kt new file mode 100644 index 0000000..709a3bc --- /dev/null +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/StreamOsdTest.kt @@ -0,0 +1,174 @@ +package com.librestatic.ezdisplay + +import androidx.activity.ComponentActivity +import android.graphics.Color +import android.view.Gravity +import android.view.View +import android.widget.FrameLayout +import androidx.test.core.app.ApplicationProvider +import com.github.takahirom.roborazzi.captureRoboImage +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.Robolectric +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.GraphicsMode + +@RunWith(RobolectricTestRunner::class) +@GraphicsMode(GraphicsMode.Mode.NATIVE) +class StreamOsdTest { + private val context get() = ApplicationProvider.getApplicationContext() + private val res get() = context.resources + + private fun status(report: String, extra: String = "") = JSONObject( + """{"codec":"h264","bitrate_kbps":46000,"width":2960,"height":1848,"refresh_mhz":120000, + "report_seq":2,"report":$report$extra}""", + ) + + private val autoReport = """{"wanted_kbps":46000,"user_bitrate":false,"ceiling_kbps":80000, + "attempts":[{"encoder":"h264_vaapi","options":"rc_mode=VBR low_power=1","ok":false,"error":"Invalid argument"}, + {"encoder":"h264_nvenc","options":"rc=vbr cq=19","ok":true,"error":""}], + "rate_mode":"vbr","quality":19,"target_kbps":46000,"max_kbps":92000}""" + + @Test fun reportWithAttemptsAndPeak() { + val c = StreamOsd.render(res, StreamOsd.parse(status(autoReport))) + assertEquals("Negotiated 46 Mbit/s · auto for 2960×1848 at 120 Hz · link limit 80 Mbit/s", c.lines[0]) + assertEquals("✗ h264_vaapi rc_mode=VBR low_power=1 — Invalid argument", c.lines[1]) + assertEquals("✓ h264_nvenc rc=vbr cq=19", c.lines[2]) + assertEquals("H.264 · variable bitrate, quality 19 · 46 Mbit/s (peak 92)", c.headline) + } + + @Test fun clampedBitrateShowsWantedAndLimit() { + val r = """{"wanted_kbps":95000,"user_bitrate":false,"ceiling_kbps":80000,"attempts":[], + "rate_mode":"cbr","quality":0,"target_kbps":80000,"max_kbps":80000}""" + val j = status(r).put("bitrate_kbps", 80000) + val c = StreamOsd.render(res, StreamOsd.parse(j)) + assertTrue(c.lines[0], c.lines[0].contains("wanted 95, limited to 80")) + assertEquals("H.264 · constant bitrate · 80 Mbit/s", c.headline) + } + + @Test fun userBitrateSaysSetInSettings() { + val r = autoReport.replace("\"user_bitrate\":false", "\"user_bitrate\":true") + val c = StreamOsd.render(res, StreamOsd.parse(status(r))) + assertTrue(c.lines[0], c.lines[0].contains("set in settings")) + assertFalse(c.lines[0].contains("auto for")) + } + + @Test fun smallBitrateHasOneDecimal() { + assertEquals("4.5 Mbit/s", StreamOsd.mbit(res, 4500)) + assertEquals("46 Mbit/s", StreamOsd.mbit(res, 46000)) + } + + @Test fun longErrorIsEllipsizedOnOneLine() { + val r = autoReport.replace("Invalid argument", "x".repeat(200) + "\\nsecond") + val line = StreamOsd.render(res, StreamOsd.parse(status(r))).lines[1] + assertTrue(line, line.endsWith("…")) + assertFalse(line.contains("\n")) + assertTrue(line.length < 120) + } + + @Test fun attemptListIsCappedWithMoreLine() { + val attempts = (1..9).joinToString(",") { """{"encoder":"e$it","options":"","ok":false,"error":""}""" } + val r = """{"wanted_kbps":1,"user_bitrate":false,"ceiling_kbps":2,"attempts":[$attempts], + "rate_mode":"unknown","quality":0,"target_kbps":1,"max_kbps":1}""" + val c = StreamOsd.render(res, StreamOsd.parse(status(r))) + assertEquals(1 + 1 + StreamOsd.MAX_ATTEMPTS, c.lines.size) + assertEquals("+3 more", c.lines[1]) + assertEquals("✗ e4", c.lines[2]) + assertEquals("✗ e9", c.lines.last()) + } + + @Test fun oldHostShowsOnlyCodecAndBitrate() { + val j = JSONObject("""{"codec":"hevc","bitrate_kbps":30000,"width":1,"height":1,"refresh_mhz":60000}""") + val info = StreamOsd.parse(j) + assertEquals(0, info.seq) + assertNull(info.report) + val c = StreamOsd.render(res, info) + assertTrue(c.lines.isEmpty()) + assertEquals("HEVC · 30 Mbit/s", c.headline) + } + + @Test fun unknownRateModeIsOmitted() { + val r = autoReport.replace("\"vbr\"", "\"unknown\"").replace("92000", "46000") + assertEquals("H.264 · 46 Mbit/s", StreamOsd.render(res, StreamOsd.parse(status(r))).headline) + } + + @Test @Config(qualifiers = "es") fun spanishResolves() { + val c = StreamOsd.render(res, StreamOsd.parse(status(autoReport))) + assertTrue(c.lines[0], c.lines[0].startsWith("Negociado: 46 Mbit/s")) + assertTrue(c.headline, c.headline.contains("tasa de bits variable, calidad 19")) + } + + private val losslessReport = """{"wanted_kbps":46000,"user_bitrate":false,"ceiling_kbps":0, + "attempts":[{"encoder":"hevc_nvenc","options":"tune=lossless","ok":false,"error":"Function not implemented"}, + {"encoder":"hevc_nvenc","options":"rc=constqp qp=10","ok":true,"error":""}], + "rate_mode":"cqp","quality":10,"target_kbps":46000,"max_kbps":0, + "wanted_quality":"lossless","applied_quality":"high","wanted_codec":"hevc"}""" + + @Test fun qualityLineShowsRequestedAndApplied() { + val j = status(losslessReport).put("codec", "hevc") + val c = StreamOsd.render(res, StreamOsd.parse(j)) + assertEquals("Quality: lossless requested · high applied", c.lines[1]) + assertEquals("HEVC · constant quality 10", c.headline) + } + + @Test fun qualityLineWhenAppliedAsWanted() { + val r = losslessReport.replace("\"applied_quality\":\"high\"", "\"applied_quality\":\"lossless\"") + .replace("\"cqp\"", "\"lossless\"") + val c = StreamOsd.render(res, StreamOsd.parse(status(r).put("codec", "hevc"))) + assertEquals("Quality: lossless", c.lines[1]) + assertEquals("HEVC · lossless", c.headline) + } + + @Test fun codecLineOnlyWhenTheFinalCodecDiffers() { + val c = StreamOsd.render(res, StreamOsd.parse(status(losslessReport))) + assertTrue(c.lines.toString(), c.lines.contains("Codec: HEVC requested")) + val same = StreamOsd.render(res, StreamOsd.parse(status(losslessReport).put("codec", "hevc"))) + assertFalse(same.lines.toString(), same.lines.any { it.startsWith("Codec:") }) + } + + @Test fun missingNewKeysAddNoLines() { + val c = StreamOsd.render(res, StreamOsd.parse(status(autoReport))) + assertEquals(3, c.lines.size) + assertFalse(c.lines.any { it.startsWith("Quality:") || it.startsWith("Codec:") }) + } + + @Test @Config(qualifiers = "es") fun spanishQualityLine() { + val c = StreamOsd.render(res, StreamOsd.parse(status(losslessReport).put("codec", "hevc"))) + assertEquals("Calidad: sin pérdida solicitada · alta aplicada", c.lines[1]) + } + + @Test fun viewIsNotInteractive() { + val v = StreamOsdView(context) + assertFalse(v.isClickable) + assertFalse(v.isFocusable) + } + + @Test @Config(qualifiers = "w1280dp-h800dp-notnight") + fun captureOsd() { + val activity = Robolectric.buildActivity(ComponentActivity::class.java).setup().get() + val root = FrameLayout(activity).apply { setBackgroundColor(Color.rgb(40, 60, 90)) } + val osd = StreamOsdView(activity) + root.addView(osd, FrameLayout.LayoutParams(-2, -2, Gravity.BOTTOM or Gravity.START).apply { setMargins(32, 32, 32, 32) }) + activity.setContentView(root) + osd.show(StreamOsd.render(activity.resources, StreamOsd.parse(status(autoReport)))) + assertEquals(View.VISIBLE, osd.visibility) + root.captureRoboImage("build/outputs/roborazzi/stream_osd.png") + } + + @Test @Config(qualifiers = "w1280dp-h800dp-notnight") + fun captureOsdLossless() { + val activity = Robolectric.buildActivity(ComponentActivity::class.java).setup().get() + val root = FrameLayout(activity).apply { setBackgroundColor(Color.rgb(40, 60, 90)) } + val osd = StreamOsdView(activity) + root.addView(osd, FrameLayout.LayoutParams(-2, -2, Gravity.BOTTOM or Gravity.START).apply { setMargins(32, 32, 32, 32) }) + activity.setContentView(root) + osd.show(StreamOsd.render(activity.resources, StreamOsd.parse(status(losslessReport).put("codec", "hevc")))) + root.captureRoboImage("build/outputs/roborazzi/stream_osd_lossless.png") + } +} diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt index b4fcd65..778965f 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsPhoneTest.kt @@ -3,6 +3,7 @@ package com.librestatic.ezdisplay.settings import androidx.compose.ui.semantics.SemanticsActions import androidx.compose.ui.semantics.SemanticsProperties import androidx.compose.ui.test.assertIsEnabled +import androidx.compose.ui.test.assertIsOn import androidx.compose.ui.test.assertIsNotEnabled import androidx.compose.ui.test.assertCountEquals import androidx.compose.ui.test.hasClickAction @@ -48,6 +49,15 @@ class SettingsPhoneTest : SettingsUiTestBase() { assertEquals("desktop", store.state.value.profile) } + @Test fun videoChoicesSaveCodecAndQuality() { + launch() + tag("video_codec_hevc").performScrollTo().performClick() + tag("stream_quality_lossless").performScrollTo().performClick() + assertEquals("hevc", prefs.getString(SettingsStore.VIDEO_CODEC, null)) + assertEquals("lossless", prefs.getString(SettingsStore.STREAM_QUALITY, null)) + tag("stream_quality_lossless").assertIsOn() + } + @Test fun selectedProfileShowsDescriptionAndMarker() { launch { putString(InputPrefs.PROFILE, "drawing") } rule.onNodeWithText(str(R.string.profile_drawing_desc)).assertExists() diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt index c506810..68d3be5 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsScreenshotTest.kt @@ -54,6 +54,8 @@ class SettingsScreenshotTest { override fun setScreensaverSpeed(level: Int) {} override fun setScreensaverMotion(mode: ScreensaverMotionMode) {} override fun setAudioEnabled(on: Boolean) {} + override fun setVideoCodec(codec: String) {} + override fun setStreamQuality(quality: String) {} override fun startPairing(name: String, shortFp: String, addr: String) {} override fun startPairing(link: PairLink) {} override fun submitPairing(pin: String) {} @@ -115,6 +117,14 @@ class SettingsScreenshotTest { deviceFingerprint = "abcd-0123-4567-89ef", ) + @Test @Config(sdk = [36], qualifiers = "w412dp-h915dp-night") + fun phone_video_dark() { + show(SettingsState(videoCodec = "hevc", streamQuality = "lossless")) + rule.onNodeWithTag("video_group").performScrollTo() + rule.waitForIdle() + capture("phone_video_dark") + } + @Test @Config(sdk = [36], qualifiers = "w412dp-h915dp-night") fun phone_connection_dark() { show(connectionState) diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt index 0fe81db..8ec05d6 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsStoreTest.kt @@ -3,6 +3,7 @@ package com.librestatic.ezdisplay.settings import android.content.Context import android.content.SharedPreferences import androidx.test.core.app.ApplicationProvider +import com.librestatic.ezdisplay.Capabilities import com.librestatic.ezdisplay.InputPrefs import com.librestatic.ezdisplay.screensaver.ScreensaverMotionMode import com.librestatic.ezdisplay.InputPrefs.KeyCombo @@ -281,6 +282,46 @@ class SettingsStoreTest { assertEquals("raw", SettingsStore(prefs, AirActions.Off).state.value.profile) } + @Test fun videoPrefsDefaultToAutoAndNormal() { + assertEquals("auto", s.videoCodec) + assertEquals("normal", s.streamQuality) + assertEquals("auto", Capabilities.streamPrefs(prefs, hevcAvailable = true).getString("codec")) + assertEquals("normal", Capabilities.streamPrefs(prefs, hevcAvailable = true).getString("quality")) + } + + @Test fun videoPrefsPersistAndReachTheCapabilities() { + store.setVideoCodec("hevc") + store.setStreamQuality("lossless") + assertEquals("hevc", s.videoCodec) + assertEquals("lossless", s.streamQuality) + val reread = SettingsStore(prefs, AirActions.Off).state.value + assertEquals("hevc", reread.videoCodec) + assertEquals("lossless", reread.streamQuality) + val j = Capabilities.streamPrefs(prefs, hevcAvailable = true) + assertEquals("hevc", j.getString("codec")) + assertEquals("lossless", j.getString("quality")) + } + + @Test fun invalidVideoValuesAreIgnored() { + store.setVideoCodec("av1") + store.setStreamQuality("ultra") + assertEquals("auto", s.videoCodec) + assertEquals("normal", s.streamQuality) + prefs.edit().putString(SettingsStore.VIDEO_CODEC, "bogus").putString(SettingsStore.STREAM_QUALITY, "x").commit() + val j = Capabilities.streamPrefs(prefs, hevcAvailable = true) + assertEquals("auto", j.getString("codec")) + assertEquals("normal", j.getString("quality")) + } + + @Test fun hevcIsRefusedWithoutADecoder() { + val noHevc = SettingsStore(prefs, AirActions.Off, hevcAvailable = false) + noHevc.setVideoCodec("hevc") + assertEquals("auto", noHevc.state.value.videoCodec) + assertFalse(noHevc.state.value.hevcAvailable) + prefs.edit().putString(SettingsStore.VIDEO_CODEC, "hevc").commit() + assertEquals("auto", Capabilities.streamPrefs(prefs, hevcAvailable = false).getString("codec")) + } + @Test fun screensaverDefaults() { assertFalse(s.screensaverEnabled) assertEquals(5, s.screensaverIdleMin) diff --git a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsWideTest.kt b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsWideTest.kt index be9b4a6..8e64bc1 100644 --- a/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsWideTest.kt +++ b/android/app/src/test/kotlin/com/librestatic/ezdisplay/settings/SettingsWideTest.kt @@ -124,6 +124,17 @@ class SettingsWideTest : SettingsUiTestBase() { } } + @Test fun videoRowOpensADialogWithTheChoices() { + launch() + tag("video_group").assertDoesNotExist() + tag("video_summary").performClick() + tag("video_dialog").assertExists() + tag("stream_quality_high").performClick() + assertEquals("high", prefs.getString(SettingsStore.STREAM_QUALITY, null)) + tag("video_dialog_close").performClick() + tag("video_dialog").assertDoesNotExist() + } + @Test fun airCardSitsBelowThePalmRow() { launch(AirActions.On) { putString(InputPrefs.PROFILE, "graphics_tablet") } val palm = bounds("palm_row") diff --git a/crates/ezd-android/src/caps.rs b/crates/ezd-android/src/caps.rs index 37cbe2c..2ef6e02 100644 --- a/crates/ezd-android/src/caps.rs +++ b/crates/ezd-android/src/caps.rs @@ -10,11 +10,17 @@ //! "hdr": {"display_hdr10": true, "max_nits": 1000, "max_avg_nits": 600, "min_millinits": 0}, //! "audio": true, //! "stylus": true, -//! "max_touch_points": 10 +//! "max_touch_points": 10, +//! "stream_prefs": {"codec": "auto", "quality": "normal"} //! } //! ``` +//! +//! `stream_prefs` is optional: `codec` is `auto`, `h264`, `hevc` or `av1`, `quality` is `normal`, +//! `high` or `lossless`; a missing object or an unknown string means `auto` / `normal`. -use ezd_proto::{ClientCaps, ClientHdr, Codec, DecoderCaps, DisplayInfo}; +use ezd_proto::{ + ClientCaps, ClientHdr, ClientPrefs, Codec, DecoderCaps, DisplayInfo, QualityLevel, +}; use serde::Deserialize; /// Parsed capabilities plus the device name. @@ -28,6 +34,16 @@ pub struct ParsedCaps { pub hdr: Option, /// The user wants the PC's audio on this device (`"audio"`, default false). pub audio: bool, + /// Codec and quality for `ClientConfig.prefs`. + pub prefs: ClientPrefs, +} + +#[derive(Deserialize, Default)] +struct JsonPrefs { + #[serde(default)] + codec: String, + #[serde(default)] + quality: String, } #[derive(Deserialize)] @@ -43,6 +59,8 @@ struct JsonCaps { audio: bool, #[serde(default)] stylus: bool, + #[serde(default)] + stream_prefs: Option, #[serde(default = "default_touch")] max_touch_points: u32, } @@ -106,6 +124,14 @@ fn parse_codec(s: &str) -> Option { } } +fn parse_quality(s: &str) -> QualityLevel { + match s.to_ascii_lowercase().as_str() { + "high" => QualityLevel::High, + "lossless" => QualityLevel::Lossless, + _ => QualityLevel::Normal, + } +} + /// Parses the Kotlin-provided capabilities JSON. Decoders of unknown codecs are skipped. pub fn parse_caps(json: &str) -> Result { let j: JsonCaps = serde_json::from_str(json).map_err(|e| format!("bad caps json: {e}"))?; @@ -148,9 +174,11 @@ pub fn parse_caps(json: &str) -> Result { &main10, ) }); + let prefs = j.stream_prefs.unwrap_or_default(); Ok(ParsedCaps { hdr, audio: j.audio, + prefs: ClientPrefs::new(parse_codec(&prefs.codec), parse_quality(&prefs.quality)), device_name: if j.device_name.is_empty() { "android".into() } else { @@ -296,6 +324,39 @@ mod tests { assert_eq!(h, ClientHdr::none()); } + #[test] + fn parses_stream_prefs() { + let with = |prefs: &str| { + let json = format!( + r#"{{"display": {{"width_px": 1, "height_px": 1, "refresh_rates_mhz": [60000]}}{prefs}}}"# + ); + parse_caps(&json).unwrap().prefs + }; + assert_eq!(with(""), ClientPrefs::auto()); + assert_eq!(with(r#", "stream_prefs": {}"#), ClientPrefs::auto()); + assert_eq!( + with(r#", "stream_prefs": {"codec": "hevc", "quality": "lossless"}"#), + ClientPrefs::new(Some(Codec::Hevc), QualityLevel::Lossless) + ); + assert_eq!( + with(r#", "stream_prefs": {"codec": "h264", "quality": "high"}"#), + ClientPrefs::new(Some(Codec::H264), QualityLevel::High) + ); + assert_eq!( + with(r#", "stream_prefs": {"codec": "av1"}"#), + ClientPrefs::new(Some(Codec::Av1), QualityLevel::Normal) + ); + // Unknown strings mean auto / normal. + assert_eq!( + with(r#", "stream_prefs": {"codec": "vp9", "quality": "ultra"}"#), + ClientPrefs::auto() + ); + assert_eq!( + with(r#", "stream_prefs": {"codec": "auto", "quality": "normal"}"#), + ClientPrefs::auto() + ); + } + #[test] fn rejects_missing_display() { assert!(parse_caps("{}").is_err()); diff --git a/crates/ezd-android/src/session.rs b/crates/ezd-android/src/session.rs index 08d1087..8473bac 100644 --- a/crates/ezd-android/src/session.rs +++ b/crates/ezd-android/src/session.rs @@ -14,7 +14,7 @@ use ezd_link::{BlockingOptions, Link}; use ezd_proto::pair::Fingerprint; use ezd_proto::{ AudioCodec, AudioConfig, AudioPacket, AudioVolume, ClientAudio, CursorShape, SampleFormat, - StreamColor, StreamConfig, VideoFrame, + StreamColor, StreamConfig, StreamReport, VideoFrame, }; use std::fs::File; use std::io::{Read, Write}; @@ -223,6 +223,11 @@ impl ClientHandler for Handler { self.color = *color; } + fn on_stream_report(&mut self, report: &StreamReport) { + tracing::info!(?report, "host stream report"); + self.shared.set_report(report.clone()); + } + fn on_stream_config(&mut self, cfg: &StreamConfig) -> HandlerResult { tracing::info!( "stream {}x{} @ {:.3} Hz {:?} {} kbps ({})", @@ -480,6 +485,7 @@ fn run( cfg.app_version = env!("CARGO_PKG_VERSION").into(); cfg.hdr = caps.hdr; cfg.audio = Some(client_audio(caps.audio)); + cfg.prefs = caps.prefs; let mut handler = Handler { shared: shared.clone(), dec: dec.clone(), diff --git a/crates/ezd-android/src/status.rs b/crates/ezd-android/src/status.rs index 0416acb..47c0b67 100644 --- a/crates/ezd-android/src/status.rs +++ b/crates/ezd-android/src/status.rs @@ -1,7 +1,7 @@ //! Session phase and the JSON returned by `pollStats()`. use ezd_client_core::ClientSnapshot; -use ezd_proto::{Codec, StreamConfig}; +use ezd_proto::{Codec, QualityLevel, StreamConfig, StreamReport}; use serde::Serialize; use std::sync::Mutex; @@ -33,6 +33,10 @@ struct State { decoder: Option, /// Size of the decoder's output buffers, which can exceed the stream (coded vs cropped size). buffer: Option<(i32, i32)>, + /// How the host negotiated and encodes the current stream, once it sent the report. + report: Option, + /// Reports stored so far in this process (0 = none yet). + report_seq: u64, } /// State shared between the client task, the decoder thread and the JNI poll calls. @@ -47,6 +51,8 @@ impl Default for Shared { stream: None, decoder: None, buffer: None, + report: None, + report_seq: 0, })) } } @@ -82,7 +88,19 @@ impl Shared { /// Records the negotiated stream. pub fn set_stream(&self, stream: StreamConfig) { - self.with(|s| s.stream = Some(stream)); + self.with(|s| { + s.stream = Some(stream); + // The report of the previous stream no longer applies. + s.report = None; + }); + } + + /// Records the host's report on the encoder setup. + pub fn set_report(&self, report: StreamReport) { + self.with(|s| { + s.report = Some(report); + s.report_seq += 1; + }); } /// Records the decoder in use. @@ -128,6 +146,30 @@ impl Shared { Codec::Av1 => "av1", }); let color = &snap.stream_color; + let report = s.report.as_ref().map(|r| { + serde_json::json!({ + "wanted_kbps": r.wanted_kbps, + "user_bitrate": r.user_bitrate, + "ceiling_kbps": r.ceiling_kbps, + "attempts": r.attempts.iter().map(|a| serde_json::json!({ + "encoder": a.encoder, + "options": a.options, + "ok": a.ok, + "error": a.error, + })).collect::>(), + "rate_mode": r.rate.rate_mode().name(), + "quality": r.rate.quality, + "target_kbps": r.rate.target_kbps, + "max_kbps": r.rate.max_kbps, + "wanted_quality": QualityLevel::from_code(r.wanted_quality).name(), + "applied_quality": QualityLevel::from_code(r.applied_quality).name(), + "wanted_codec": Codec::from_code(r.wanted_codec).map_or("auto", |c| match c { + Codec::H264 => "h264", + Codec::Hevc => "hevc", + Codec::Av1 => "av1", + }), + }) + }); let v = serde_json::json!({ "phase": s.phase, "detail": s.detail, @@ -160,6 +202,8 @@ impl Shared { "input_note": snap.input_status.as_ref().map(|i| i.note.clone()), "client_flags": snap.client_options.map(|o| o.flags), "inputs_sent": snap.inputs_sent, + "report_seq": s.report_seq, + "report": report, }); v.to_string() } @@ -194,6 +238,67 @@ mod tests { assert_eq!(j["bit_depth"], 8); } + #[test] + fn json_has_the_stream_report_and_drops_it_with_the_stream() { + use ezd_proto::{EncoderAttempt, RateControl, RateMode}; + let sh = Shared::default(); + let j = |sh: &Shared| -> serde_json::Value { + serde_json::from_str(&sh.to_json(&Default::default())).unwrap() + }; + assert_eq!(j(&sh)["report_seq"], 0); + assert!(j(&sh)["report"].is_null()); + sh.set_report(StreamReport { + wanted_kbps: 45_949, + user_bitrate: false, + ceiling_kbps: 200_000, + attempts: vec![EncoderAttempt { + encoder: "h264_nvenc".into(), + options: "rc=vbr cq=19".into(), + ok: true, + error: String::new(), + }], + rate: RateControl { + mode: RateMode::Vbr.code(), + quality: 19, + target_kbps: 45_949, + max_kbps: 91_898, + }, + wanted_quality: QualityLevel::Lossless.code(), + applied_quality: QualityLevel::High.code(), + wanted_codec: Codec::Hevc.code(), + }); + let v = j(&sh); + assert_eq!(v["report_seq"], 1); + let r = &v["report"]; + assert_eq!(r["wanted_kbps"], 45_949); + assert_eq!(r["user_bitrate"], false); + assert_eq!(r["ceiling_kbps"], 200_000); + assert_eq!(r["attempts"][0]["encoder"], "h264_nvenc"); + assert_eq!(r["attempts"][0]["options"], "rc=vbr cq=19"); + assert_eq!(r["attempts"][0]["ok"], true); + assert_eq!(r["attempts"][0]["error"], ""); + assert_eq!(r["rate_mode"], "vbr"); + assert_eq!(r["quality"], 19); + assert_eq!(r["wanted_quality"], "lossless"); + assert_eq!(r["applied_quality"], "high"); + assert_eq!(r["wanted_codec"], "hevc"); + assert_eq!( + (r["target_kbps"].as_u64(), r["max_kbps"].as_u64()), + (Some(45_949), Some(91_898)) + ); + sh.set_stream(StreamConfig { + width: 1, + height: 1, + refresh_mhz: 60_000, + codec: Codec::H264, + bitrate_kbps: 1, + encoder_name: "x".into(), + }); + let v = j(&sh); + assert!(v["report"].is_null()); + assert_eq!(v["report_seq"], 1); + } + #[test] fn json_reports_hdr10() { let sh = Shared::default(); diff --git a/crates/ezd-client-core/src/session.rs b/crates/ezd-client-core/src/session.rs index e0afece..9c03c5a 100644 --- a/crates/ezd-client-core/src/session.rs +++ b/crates/ezd-client-core/src/session.rs @@ -3,8 +3,9 @@ use crate::{ClientHandle, now_us}; use ezd_link::{Link, MuxError, MuxReceiver, MuxSender}; use ezd_proto::{ - AudioConfig, AudioPacket, AudioVolume, Bye, ClientAudio, ClientCaps, ClientHdr, CursorShape, - CursorState, FrameAck, Hello, Message, Ping, Pong, Role, StreamColor, StreamConfig, VideoFrame, + AudioConfig, AudioPacket, AudioVolume, Bye, ClientAudio, ClientCaps, ClientHdr, ClientPrefs, + CursorShape, CursorState, FrameAck, Hello, Message, Ping, Pong, Role, StreamColor, + StreamConfig, StreamReport, VideoFrame, }; use std::time::Duration; use tokio::time::{MissedTickBehavior, interval, timeout}; @@ -62,6 +63,10 @@ pub trait ClientHandler: Send { /// `received_us` is its [`now_us`] at reassembly. fn on_video_frame(&mut self, frame: VideoFrame, received_us: u64) -> HandlerResult; + /// How the host negotiated and encodes the stream (protocol 0.7). Sent after the encoder + /// opens, before the first frame it produced, and again after every reopen. + fn on_stream_report(&mut self, _report: &StreamReport) {} + /// A new cursor shape (already cached in the handle). fn on_cursor_shape(&mut self, _shape: &CursorShape) {} @@ -86,6 +91,9 @@ pub struct ClientConfig { /// right after `ClientHdr`; `None` sends [`ClientAudio::none`] (no audio), so the host /// always gets an answer and never waits for it in vain. pub audio: Option, + /// Codec and quality asked for in `ClientPrefs`, sent right after `ClientHdr` whenever the + /// host speaks protocol 0.8. The default is [`ClientPrefs::auto`]. + pub prefs: ClientPrefs, /// Interval of `ClientStats` messages. pub stats_interval: Duration, /// Interval of client-initiated pings (clock sync). @@ -112,6 +120,7 @@ impl ClientConfig { caps, hdr: None, audio: None, + prefs: ClientPrefs::auto(), stats_interval: Duration::from_secs(1), ping_interval: Duration::from_millis(500), max_duration: None, @@ -245,6 +254,7 @@ pub async fn run_client( let my_minor_supports_hdr = my_hello.supports_hdr(); let my_minor_supports_audio = my_hello.supports_audio(); let my_minor_supports_keyboard = my_hello.supports_keyboard(); + let my_minor_supports_prefs = my_hello.supports_stream_prefs(); tx.send_control(Message::Hello(my_hello))?; tx.send_control(Message::ClientCaps(cfg.caps.clone()))?; let hello = wait_for(&mut rx, &tx, "Hello", wait, |m| match m { @@ -265,6 +275,9 @@ pub async fn run_client( cfg.audio.clone().unwrap_or_else(ClientAudio::none), ))?; } + if hello.supports_stream_prefs() && my_minor_supports_prefs { + tx.send_control(Message::ClientPrefs(cfg.prefs))?; + } wait_for(&mut rx, &tx, "HostCaps", wait, |m| match m { Message::HostCaps(h) => Ok(h), o => Err(o), @@ -461,6 +474,10 @@ fn on_message( handle.count_received(v.data.len()); flow(handler.on_video_frame(v, received_us)) } + Message::StreamReport(r) => { + handler.on_stream_report(&r); + Ok(None) + } Message::AudioPacket(p) => { handler.on_audio_packet(p); Ok(None) diff --git a/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl index 7c8d0ef..747d6a5 100644 --- a/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/de/ezd-fake-client.ftl @@ -17,6 +17,8 @@ arg-max-seconds = Bricht nach dieser Anzahl von Sekunden ab. arg-input-script = Spielt einmal eine vorgefertigte Eingabesequenz ab, beginnend mit dem ersten Videobild: tap, drag, circle, touch2, scroll oder keys (Strg+Leertaste während eines Stiftstrichs gehalten; benötigt einen Host mit KEYS). arg-exit-after-script = Beendet kurz nach Ende des Eingabeskripts. arg-audio = Meldet die Wiedergabe des PC-Audios: Der Host streamt sein Audio, die Pakete werden gezählt (nicht abgespielt). +arg-codec = Beim Host angefragter Codec: auto, h264, hevc oder av1. Der Host nutzt ihn nur, wenn beide Seiten ihn unterstützen. +arg-quality = Beim Host angefragte Qualität: normal, high oder lossless. Der Host setzt high und lossless nur bei einer USB-Verbindung um; lossless benötigt HEVC. err-dump-frame = --dump-frame N muss eine Zahl sein err-connecting = Verbindung zu { $addr } diff --git a/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl index 0d782c7..51c50ee 100644 --- a/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/en/ezd-fake-client.ftl @@ -17,6 +17,8 @@ arg-max-seconds = Abort after this many seconds. arg-input-script = Play a canned input sequence once, starting at the first video frame: tap, drag, circle, touch2, scroll, or keys (Ctrl+Space held during a pen stroke; needs a host with KEYS). arg-exit-after-script = Exit shortly after the input script finished. arg-audio = Advertise PC audio playback: the host streams its audio and the packets are counted (not played). +arg-codec = Codec to ask the host for: auto, h264, hevc or av1. The host uses it only if both sides support it. +arg-quality = Quality to ask the host for: normal, high or lossless. The host honours high and lossless only on a USB link; lossless needs HEVC. err-dump-frame = --dump-frame N must be a number err-connecting = connecting to { $addr } diff --git a/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl index 1076a26..a1c713e 100644 --- a/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/es/ezd-fake-client.ftl @@ -17,6 +17,8 @@ arg-max-seconds = Aborta tras este número de segundos. arg-input-script = Reproduce una vez una secuencia de entrada predefinida, desde el primer fotograma de video: tap, drag, circle, touch2, scroll o keys (Ctrl+Espacio mantenido durante un trazo del lápiz; requiere un host con KEYS). arg-exit-after-script = Termina poco después de acabar el guion de entrada. arg-audio = Anuncia la reproducción del audio de la PC: el host envía su audio y los paquetes se cuentan (no se reproducen). +arg-codec = Códec que se le pide al host: auto, h264, hevc o av1. El host lo usa solo si ambos lo soportan. +arg-quality = Calidad que se le pide al host: normal, high o lossless. El host respeta high y lossless solo en un enlace USB; lossless requiere HEVC. err-dump-frame = --dump-frame N debe ser un número err-connecting = conectando con { $addr } diff --git a/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl index 4d8a306..371466d 100644 --- a/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/fr/ezd-fake-client.ftl @@ -17,6 +17,8 @@ arg-max-seconds = Abandonne après ce nombre de secondes. arg-input-script = Joue une fois une séquence d'entrée prédéfinie, dès la première image vidéo : tap, drag, circle, touch2, scroll ou keys (Ctrl+Espace maintenu pendant un tracé du stylet ; nécessite un hôte avec KEYS). arg-exit-after-script = Quitte peu après la fin du script d'entrée. arg-audio = Annonce la lecture de l'audio du PC : l'hôte diffuse son audio et les paquets sont comptés (pas lus). +arg-codec = Codec demandé à l'hôte : auto, h264, hevc ou av1. L'hôte ne l'utilise que si les deux côtés le prennent en charge. +arg-quality = Qualité demandée à l'hôte : normal, high ou lossless. L'hôte n'applique high et lossless que sur une liaison USB ; lossless nécessite HEVC. err-dump-frame = --dump-frame N doit être un nombre err-connecting = connexion à { $addr } diff --git a/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl index 8624cef..f39ec27 100644 --- a/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/it/ezd-fake-client.ftl @@ -17,6 +17,8 @@ arg-max-seconds = Interrompe dopo questo numero di secondi. arg-input-script = Riproduce una volta una sequenza di input predefinita, a partire dal primo fotogramma video: tap, drag, circle, touch2, scroll o keys (Ctrl+Spazio tenuto durante un tratto della penna; richiede un host con KEYS). arg-exit-after-script = Esce poco dopo la fine dello script di input. arg-audio = Annuncia la riproduzione dell'audio del PC: l'host trasmette il suo audio e i pacchetti vengono contati (non riprodotti). +arg-codec = Codec richiesto all'host: auto, h264, hevc o av1. L'host lo usa solo se entrambi i lati lo supportano. +arg-quality = Qualità richiesta all'host: normal, high o lossless. L'host applica high e lossless solo su un collegamento USB; lossless richiede HEVC. err-dump-frame = --dump-frame N deve essere un numero err-connecting = connessione a { $addr } diff --git a/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl b/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl index 577c4e0..a7ee4be 100644 --- a/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl +++ b/crates/ezd-fake-client/i18n/pt/ezd-fake-client.ftl @@ -17,6 +17,8 @@ arg-max-seconds = Aborta após este número de segundos. arg-input-script = Reproduz uma vez uma sequência de entrada predefinida, a partir do primeiro quadro de vídeo: tap, drag, circle, touch2, scroll ou keys (Ctrl+Espaço mantido durante um traço da caneta; requer um host com KEYS). arg-exit-after-script = Sai logo após o término do roteiro de entrada. arg-audio = Anuncia a reprodução do áudio do PC: o host envia o áudio e os pacotes são contados (não reproduzidos). +arg-codec = Codec solicitado ao host: auto, h264, hevc ou av1. O host só o usa se os dois lados o suportarem. +arg-quality = Qualidade solicitada ao host: normal, high ou lossless. O host só respeita high e lossless em um link USB; lossless requer HEVC. err-dump-frame = --dump-frame N deve ser um número err-connecting = conectando a { $addr } diff --git a/crates/ezd-fake-client/src/lib.rs b/crates/ezd-fake-client/src/lib.rs index 17c8f92..f5edc42 100644 --- a/crates/ezd-fake-client/src/lib.rs +++ b/crates/ezd-fake-client/src/lib.rs @@ -20,8 +20,8 @@ use ezd_host_core::synthetic::read_frame_number; use ezd_link::{Link, MuxError}; use ezd_proto::{ AudioCodec, AudioConfig, AudioPacket, AudioStream, ClientAudio, ClientCaps, ClientOptions, - Codec, CursorShape, CursorState, DecoderCaps, DisplayInfo, InputStatus, SampleFormat, - StreamConfig, VideoFrame, + ClientPrefs, Codec, CursorShape, CursorState, DecoderCaps, DisplayInfo, InputStatus, + QualityLevel, SampleFormat, StreamConfig, StreamReport, VideoFrame, }; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -67,6 +67,8 @@ pub struct FakeClientConfig { pub audio: bool, /// Hardware keyboard state to report with `ClientKeyboard` (`None` = never set). pub keyboard_attached: Option, + /// Codec and quality to ask the host for (`ClientPrefs`). + pub prefs: ClientPrefs, } impl Default for FakeClientConfig { @@ -89,6 +91,7 @@ impl Default for FakeClientConfig { proto_minor: None, audio: false, keyboard_attached: None, + prefs: ClientPrefs::auto(), } } } @@ -378,6 +381,16 @@ impl Handler { } impl ClientHandler for Handler { + fn on_stream_report(&mut self, report: &StreamReport) { + tracing::info!( + ?report, + wanted_quality = QualityLevel::from_code(report.wanted_quality).name(), + applied_quality = QualityLevel::from_code(report.applied_quality).name(), + wanted_codec = report.wanted_codec, + "host stream report" + ); + } + fn on_stream_config(&mut self, cfg: &StreamConfig) -> HandlerResult { self.report.stream = cfg.clone(); self.decoder = Some(Decoder::new(cfg.codec).map_err(boxed)?); @@ -475,6 +488,7 @@ pub async fn run_fake_client( core_cfg.proto_minor = cfg.proto_minor; core_cfg.audio = Some(cfg.client_audio()); + core_cfg.prefs = cfg.prefs; let handle = ClientHandle::new(); if let Some(a) = cfg.keyboard_attached { diff --git a/crates/ezd-fake-client/src/main.rs b/crates/ezd-fake-client/src/main.rs index 4a3a37c..4a2f968 100644 --- a/crates/ezd-fake-client/src/main.rs +++ b/crates/ezd-fake-client/src/main.rs @@ -6,6 +6,7 @@ use ezd_fake_client::fl; use ezd_fake_client::i18n; use ezd_fake_client::{FakeClientConfig, InputScript, run_fake_client}; use ezd_link::Link; +use ezd_proto::{ClientPrefs, Codec, QualityLevel}; use i18n_embed::fluent::FluentLanguageLoader; use std::path::PathBuf; use std::time::{Duration, Instant}; @@ -40,6 +41,10 @@ struct Args { exit_after_script: bool, #[arg(long)] audio: bool, + #[arg(long, value_enum, default_value_t = CodecArg::Auto)] + codec: CodecArg, + #[arg(long, value_enum, default_value_t = QualityArg::Normal)] + quality: QualityArg, #[arg(long, hide = true)] proto_minor: Option, #[cfg(feature = "wifi")] @@ -66,6 +71,21 @@ enum ScriptArg { Keys, } +#[derive(Debug, Clone, Copy, ValueEnum)] +enum CodecArg { + Auto, + H264, + Hevc, + Av1, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +enum QualityArg { + Normal, + High, + Lossless, +} + /// The command line with every help text taken from `l`. fn command(l: &FluentLanguageLoader) -> Command { use i18n_embed_fl::fl; @@ -88,7 +108,9 @@ fn command(l: &FluentLanguageLoader) -> Command { .mut_arg("exit_after_script", |a| { a.help(fl!(l, "arg-exit-after-script")) }) - .mut_arg("audio", |a| a.help(fl!(l, "arg-audio"))); + .mut_arg("audio", |a| a.help(fl!(l, "arg-audio"))) + .mut_arg("codec", |a| a.help(fl!(l, "arg-codec"))) + .mut_arg("quality", |a| a.help(fl!(l, "arg-quality"))); #[cfg(feature = "wifi")] let cmd = cmd .mut_arg("wifi", |a| a.help(fl!(l, "arg-wifi"))) @@ -140,6 +162,19 @@ async fn main() -> Result<()> { stop_after_script: a.exit_after_script, proto_minor: a.proto_minor, audio: a.audio, + prefs: ClientPrefs::new( + match a.codec { + CodecArg::Auto => None, + CodecArg::H264 => Some(Codec::H264), + CodecArg::Hevc => Some(Codec::Hevc), + CodecArg::Av1 => Some(Codec::Av1), + }, + match a.quality { + QualityArg::Normal => QualityLevel::Normal, + QualityArg::High => QualityLevel::High, + QualityArg::Lossless => QualityLevel::Lossless, + }, + ), ..Default::default() }; diff --git a/crates/ezd-ffmpeg/src/encoder.rs b/crates/ezd-ffmpeg/src/encoder.rs index cc21e4c..285135d 100644 --- a/crates/ezd-ffmpeg/src/encoder.rs +++ b/crates/ezd-ffmpeg/src/encoder.rs @@ -2,6 +2,7 @@ use crate::color::{ColorInfo, HdrMeta, tag_frame}; use crate::error::{EAGAIN, FfmpegError, averror_eof, init_logging}; use crate::hw::HwDevice; use crate::options::option_sets_for; +use ezd_proto::QualityLevel; use ffmpeg_sys_next as ffi; use std::ffi::{CStr, CString}; use std::ptr; @@ -67,6 +68,8 @@ pub struct EncoderConfig { pub hdr: Option, /// Extra AVOptions applied on top of the preset (override it). pub extra_options: Vec<(String, String)>, + /// Quality level selecting the option sets (default [`QualityLevel::Normal`]). + pub quality: QualityLevel, } impl EncoderConfig { @@ -89,6 +92,7 @@ impl EncoderConfig { color: ColorInfo::default(), hdr: None, extra_options: Vec::new(), + quality: QualityLevel::Normal, } } } @@ -157,7 +161,7 @@ impl Encoder { ))); } let mut last = FfmpegError::msg("no option set"); - for set in option_sets_for(&cfg.name, cfg.pix_fmt == RawPixelFormat::P010) { + for set in option_sets_for(&cfg.name, cfg.pix_fmt == RawPixelFormat::P010, cfg.quality) { match Self::try_open(codec, cfg.clone(), &set) { Ok(e) => return Ok(e), Err(e) => { @@ -200,8 +204,6 @@ impl Encoder { sw_fmt, }; let cfg = &enc.cfg; - let bitrate = cfg.bitrate_kbps as i64 * 1000; - let fps = cfg.refresh_mhz as f64 / 1000.0; // SAFETY: ctx is a freshly allocated, exclusively owned AVCodecContext; we only write plain fields. unsafe { let c = &mut *ctx; @@ -219,10 +221,10 @@ impl Encoder { Some(h) => h.kind.pix_fmt(), None => cfg.pix_fmt.av(), }; - c.bit_rate = bitrate; - c.rc_max_rate = bitrate; - // Two frames worth of VBV buffer: tight enough for low latency, loose enough for keyframes. - c.rc_buffer_size = ((bitrate as f64 * 2.0 / fps) as i32).max(1000); + let rate = crate::rate::rate_settings(cfg.bitrate_kbps, cfg.refresh_mhz); + c.bit_rate = rate.bit_rate; + c.rc_max_rate = rate.max_rate; + c.rc_buffer_size = rate.buffer_size; c.gop_size = i32::MAX; c.max_b_frames = 0; // Hardware encoders gain nothing from threads. libx264 needs them: one thread manages diff --git a/crates/ezd-ffmpeg/src/lib.rs b/crates/ezd-ffmpeg/src/lib.rs index d6cdce3..8accc60 100644 --- a/crates/ezd-ffmpeg/src/lib.rs +++ b/crates/ezd-ffmpeg/src/lib.rs @@ -24,6 +24,7 @@ mod error; mod hw; mod options; mod probe; +mod rate; mod source; #[cfg(target_os = "linux")] mod vulkan; @@ -39,6 +40,10 @@ pub use options::{option_sets, option_sets_for}; pub use probe::{ ProbeResult, encoder_available, probe_encoder, probe_encoder_with, software_encoders, }; +pub use rate::{ + HIGH_QP, NVENC_CQ, PEAK_FACTOR, RateSettings, VAAPI_QP, VBV_FRAMES, X264_CRF, describe_rate, + options_label, rate_settings, +}; pub use source::SwVideoSource; #[cfg(target_os = "linux")] pub use vulkan::vulkan_queue_families; diff --git a/crates/ezd-ffmpeg/src/options.rs b/crates/ezd-ffmpeg/src/options.rs index 7633d1e..cb4a4ab 100644 --- a/crates/ezd-ffmpeg/src/options.rs +++ b/crates/ezd-ffmpeg/src/options.rs @@ -1,16 +1,46 @@ -//! Low-latency option presets per encoder name. +//! Low-latency option presets per encoder name and quality level. + +use crate::rate::HIGH_QP; +use ezd_proto::QualityLevel; /// Ordered fallback chain of AVOption sets for `name`: the first set that /// opens successfully is used (e.g. VAAPI with `low_power` first, then without). -pub fn option_sets(name: &str) -> Vec> { - option_sets_for(name, false) +/// +/// [`QualityLevel::Normal`] is the rate-controlled preset sized to the link. +/// [`QualityLevel::High`] is a fixed quantiser of [`HIGH_QP`] (CRF for libx264). Lossless is the +/// `hevc_nvenc` lossless tune followed by the high sets, so a failed lossless open falls back to +/// high quality; every other encoder gets the high sets. +pub fn option_sets(name: &str, quality: QualityLevel) -> Vec> { + option_sets_for(name, false, quality) } /// Like [`option_sets`] for 10-bit input (P010): the HEVC encoders get `profile=main10`, and /// `hevc_vaapi` loses its `low_power` set (Gen 9.5 has no low-power HEVC Main10 and some /// drivers open it anyway). Other encoders are unchanged. -pub fn option_sets_for(name: &str, ten_bit: bool) -> Vec> { - let mut sets = base_option_sets(name); +pub fn option_sets_for( + name: &str, + ten_bit: bool, + quality: QualityLevel, +) -> Vec> { + let mut sets = match quality { + QualityLevel::Normal => normal_option_sets(name), + QualityLevel::High => high_option_sets(name), + QualityLevel::Lossless => { + let mut sets = Vec::new(); + if name == "hevc_nvenc" { + sets.push(vec![ + ("preset", "p1".to_string()), + ("tune", "lossless".to_string()), + ("zerolatency", "1".to_string()), + ("delay", "0".to_string()), + ("bf", "0".to_string()), + ("forced-idr", "1".to_string()), + ]); + } + sets.extend(high_option_sets(name)); + sets + } + }; if ten_bit && matches!(name, "hevc_nvenc" | "hevc_qsv" | "hevc_amf" | "hevc_vaapi") { if name == "hevc_vaapi" { sets.retain(|set| !set.iter().any(|(k, _)| *k == "low_power")); @@ -22,12 +52,13 @@ pub fn option_sets_for(name: &str, ten_bit: bool) -> Vec Vec> { +fn normal_option_sets(name: &str) -> Vec> { let s = |k: &'static str, v: &str| (k, v.to_string()); match name { "libx264" => vec![vec![ s("preset", "ultrafast"), s("tune", "zerolatency"), + s("crf", &crate::rate::X264_CRF.to_string()), s("forced-idr", "1"), // Keep in-band SPS/PPS on keyframes in addition to the global-header extradata. s( @@ -42,11 +73,13 @@ fn base_option_sets(name: &str) -> Vec> { s("zerolatency", "1"), s("delay", "0"), s("bf", "0"), - s("rc", "cbr"), + // Constant quality inside the VBV: a sharp first IDR and few bits on still content. + s("rc", "vbr"), + s("cq", &crate::rate::NVENC_CQ.to_string()), s("forced-idr", "1"), ]], // Level 5.2: FFmpeg picks level 6.x above 5.2's macroblock rate (2960x1848 at 120 Hz), - // and Intel's iHD driver then fails every CBR frame ("internal encoding error") on + // and Intel's iHD driver then fails every rate-controlled frame ("internal encoding error") on // Gen 9.5. 5.2 still covers the frame size and buffering, and the tablet's decoder // accepts it. // @@ -57,31 +90,31 @@ fn base_option_sets(name: &str) -> Vec> { // frame from queueing up behind the link. "h264_vaapi" => vec![ vec![ - s("rc_mode", "CBR"), + s("rc_mode", "VBR"), s("low_power", "1"), s("async_depth", "1"), s("level", "5.2"), ], vec![ s("rc_mode", "CQP"), - s("qp", "24"), + s("qp", &crate::rate::VAAPI_QP.to_string()), s("low_power", "1"), s("async_depth", "1"), s("level", "5.2"), ], vec![ - s("rc_mode", "CBR"), + s("rc_mode", "VBR"), s("async_depth", "1"), s("level", "5.2"), ], ], "hevc_vaapi" | "av1_vaapi" => vec![ vec![ - s("rc_mode", "CBR"), + s("rc_mode", "VBR"), s("low_power", "1"), s("async_depth", "1"), ], - vec![s("rc_mode", "CBR"), s("async_depth", "1")], + vec![s("rc_mode", "VBR"), s("async_depth", "1")], ], "h264_qsv" | "hevc_qsv" | "av1_qsv" => vec![ vec![ @@ -101,12 +134,16 @@ fn base_option_sets(name: &str) -> Vec> { let sets: [&[(&'static str, &str)]; 6] = [ &[ ("usage", "ultralowlatency"), - ("rc", "cbr"), + ("rc", "vbr_peak"), + ("quality", "speed"), + ], + &[ + ("usage", "lowlatency"), + ("rc", "vbr_peak"), ("quality", "speed"), ], - &[("usage", "lowlatency"), ("rc", "cbr"), ("quality", "speed")], &[("usage", "lowlatency"), ("quality", "speed")], - &[("rc", "cbr"), ("quality", "speed")], + &[("rc", "vbr_peak"), ("quality", "speed")], &[("quality", "speed")], &[], ]; @@ -123,10 +160,86 @@ fn base_option_sets(name: &str) -> Vec> { } } +/// Near-transparent constant-quantiser sets. Encoders without a verified fixed-quantiser mode +/// keep their normal sets: QSV needs `AV_CODEC_FLAG_QSCALE` for CQP, which the encoder context +/// does not set, and its ICQ mode is not available on every driver. +fn high_option_sets(name: &str) -> Vec> { + let s = |k: &'static str, v: &str| (k, v.to_string()); + let qp = HIGH_QP.to_string(); + match name { + "libx264" => vec![vec![ + s("preset", "ultrafast"), + s("tune", "zerolatency"), + s("crf", &qp), + s("forced-idr", "1"), + s( + "x264-params", + "repeat-headers=1:scenecut=0:rc-lookahead=0:sliced-threads=1", + ), + ]], + "h264_nvenc" | "hevc_nvenc" | "av1_nvenc" => vec![vec![ + s("preset", "p1"), + s("tune", "ull"), + s("zerolatency", "1"), + s("delay", "0"), + s("bf", "0"), + s("rc", "constqp"), + s("qp", &qp), + s("forced-idr", "1"), + ]], + // The same low-power first order as the normal sets (see `normal_option_sets`). + "h264_vaapi" => [true, false] + .into_iter() + .map(|low_power| { + let mut v = vec![s("rc_mode", "CQP"), s("qp", &qp)]; + if low_power { + v.push(s("low_power", "1")); + } + v.push(s("async_depth", "1")); + v.push(s("level", "5.2")); + v + }) + .collect(), + "hevc_vaapi" | "av1_vaapi" => [true, false] + .into_iter() + .map(|low_power| { + let mut v = vec![s("rc_mode", "CQP"), s("qp", &qp)]; + if low_power { + v.push(s("low_power", "1")); + } + v.push(s("async_depth", "1")); + v + }) + .collect(), + // Fixed quantiser only: the sets without `rc` would run the driver's default rate control. + "h264_amf" | "hevc_amf" | "av1_amf" => { + let sets: [&[(&'static str, &str)]; 3] = [ + &[("usage", "ultralowlatency"), ("quality", "speed")], + &[("usage", "lowlatency"), ("quality", "speed")], + &[("quality", "speed")], + ]; + sets.iter() + .map(|set| { + let mut v: Vec<_> = set.iter().map(|&(k, val)| s(k, val)).collect(); + v.push(s("rc", "cqp")); + v.push(s("qp_i", &qp)); + v.push(s("qp_p", &qp)); + v.push(s("forced_idr", "1")); + v.push(s("async_depth", "1")); + v + }) + .collect() + } + _ => normal_option_sets(name), + } +} + #[cfg(test)] mod tests { use super::*; + const N: QualityLevel = QualityLevel::Normal; + #[test] fn every_known_encoder_has_a_preset() { for n in [ @@ -137,15 +250,33 @@ mod tests { "h264_amf", "libopenh264", ] { - let sets = option_sets(n); + let sets = option_sets(n, N); assert!(!sets.is_empty() && !sets[0].is_empty(), "{n}"); } - assert_eq!(option_sets("unknown_enc"), vec![vec![]]); - assert_eq!(option_sets("h264_vaapi").len(), 3); - assert_eq!(option_sets("hevc_vaapi").len(), 2); - assert_eq!(option_sets("h264_amf").len(), 6); + let has = |n: &str, kv: (&'static str, &str)| { + option_sets(n, N) + .iter() + .any(|set| set.contains(&(kv.0, kv.1.to_string()))) + }; + for n in ["h264_nvenc", "hevc_nvenc", "av1_nvenc"] { + assert!(has(n, ("rc", "vbr")) && has(n, ("cq", "19")), "{n}"); + assert!(!has(n, ("rc", "cbr")), "{n}"); + } + assert!(has("libx264", ("crf", "20"))); + for n in ["h264_vaapi", "hevc_vaapi", "av1_vaapi"] { + assert!( + has(n, ("rc_mode", "VBR")) && !has(n, ("rc_mode", "CBR")), + "{n}" + ); + } + assert!(has("h264_vaapi", ("qp", "20")) && !has("h264_vaapi", ("qp", "24"))); + assert!(has("h264_amf", ("rc", "vbr_peak")) && !has("h264_amf", ("rc", "cbr"))); + assert_eq!(option_sets("unknown_enc", N), vec![vec![]]); + assert_eq!(option_sets("h264_vaapi", N).len(), 3); + assert_eq!(option_sets("hevc_vaapi", N).len(), 2); + assert_eq!(option_sets("h264_amf", N).len(), 6); assert!( - option_sets("h264_amf") + option_sets("h264_amf", N) .iter() .all(|set| set.contains(&("async_depth", "1".to_string()))) ); @@ -155,30 +286,116 @@ mod tests { fn ten_bit_sets_select_main10() { let main10 = ("profile", "main10".to_string()); for n in ["hevc_nvenc", "hevc_qsv", "hevc_amf", "hevc_vaapi"] { - let plain = option_sets_for(n, false); - assert_eq!(plain, option_sets(n), "{n}"); + let plain = option_sets_for(n, false, N); + assert_eq!(plain, option_sets(n, N), "{n}"); assert!(plain.iter().all(|set| !set.contains(&main10)), "{n}"); - let ten = option_sets_for(n, true); + let ten = option_sets_for(n, true, N); assert!( !ten.is_empty() && ten.iter().all(|set| set.contains(&main10)), "{n}" ); } // Counts: AMF and QSV keep their fallback chains; VAAPI drops low_power. - assert_eq!(option_sets_for("hevc_amf", true).len(), 6); - assert_eq!(option_sets_for("hevc_qsv", true).len(), 2); - assert_eq!(option_sets_for("hevc_vaapi", true).len(), 1); + assert_eq!(option_sets_for("hevc_amf", true, N).len(), 6); + assert_eq!(option_sets_for("hevc_qsv", true, N).len(), 2); + assert_eq!(option_sets_for("hevc_vaapi", true, N).len(), 1); assert!( - option_sets_for("hevc_vaapi", true) + option_sets_for("hevc_vaapi", true, N) .iter() .flatten() .all(|(k, _)| *k != "low_power") ); // Unrelated encoders ignore the flag. assert_eq!( - option_sets_for("h264_nvenc", true), - option_sets("h264_nvenc") + option_sets_for("h264_nvenc", true, N), + option_sets("h264_nvenc", N) ); - assert_eq!(option_sets_for("av1_vaapi", true), option_sets("av1_vaapi")); + assert_eq!( + option_sets_for("av1_vaapi", true, N), + option_sets("av1_vaapi", N) + ); + } + + fn has(sets: &[Vec<(&'static str, String)>], k: &str, v: &str) -> bool { + sets.iter() + .all(|set| set.contains(&(k_static(k), v.to_string()))) + } + + fn k_static(k: &str) -> &'static str { + [ + "rc", + "qp", + "tune", + "crf", + "rc_mode", + "qp_i", + "qp_p", + "low_power", + ] + .into_iter() + .find(|c| *c == k) + .unwrap() + } + + #[test] + fn high_sets_use_a_fixed_quantiser() { + let h = QualityLevel::High; + for n in ["h264_nvenc", "hevc_nvenc", "av1_nvenc"] { + let sets = option_sets(n, h); + assert_eq!(sets.len(), 1, "{n}"); + assert!(has(&sets, "rc", "constqp") && has(&sets, "qp", "10"), "{n}"); + assert!(has(&sets, "tune", "ull"), "{n}"); + assert!( + sets[0].contains(&("zerolatency", "1".into())) + && sets[0].contains(&("forced-idr", "1".into())) + ); + } + assert!(has(&option_sets("libx264", h), "crf", "10")); + for n in ["h264_vaapi", "hevc_vaapi", "av1_vaapi"] { + let sets = option_sets(n, h); + assert_eq!(sets.len(), 2, "{n}"); + assert!( + has(&sets, "rc_mode", "CQP") && has(&sets, "qp", "10"), + "{n}" + ); + assert!(sets[0].contains(&("low_power", "1".into())), "{n}"); + assert!(!sets[1].iter().any(|(k, _)| *k == "low_power"), "{n}"); + } + let amf = option_sets("h264_amf", h); + assert!(has(&amf, "rc", "cqp") && has(&amf, "qp_i", "10") && has(&amf, "qp_p", "10")); + assert!(amf.iter().all(|s| s.contains(&("async_depth", "1".into())))); + // QSV keeps its normal sets. + assert_eq!(option_sets("h264_qsv", h), option_sets("h264_qsv", N)); + // Ten-bit still selects Main10 for the fixed-quantiser sets. + let ten = option_sets_for("hevc_vaapi", true, h); + assert_eq!(ten.len(), 1); + assert!(ten[0].contains(&("profile", "main10".into()))); + } + + #[test] + fn lossless_is_an_hevc_nvenc_tune_with_high_fallback() { + let l = QualityLevel::Lossless; + let sets = option_sets("hevc_nvenc", l); + assert_eq!(sets.len(), 2); + assert!(sets[0].contains(&("tune", "lossless".into()))); + assert!( + !sets[0] + .iter() + .any(|(k, _)| matches!(*k, "rc" | "qp" | "cq")) + ); + for key in ["preset", "zerolatency", "delay", "bf", "forced-idr"] { + assert!(sets[0].iter().any(|(k, _)| *k == key), "{key}"); + } + assert_eq!(sets[1..], option_sets("hevc_nvenc", QualityLevel::High)[..]); + // Other encoders (including H.264 NVENC) have no lossless mode: the high sets. + for n in [ + "h264_nvenc", + "hevc_vaapi", + "libx264", + "hevc_amf", + "hevc_qsv", + ] { + assert_eq!(option_sets(n, l), option_sets(n, QualityLevel::High), "{n}"); + } } } diff --git a/crates/ezd-ffmpeg/src/rate.rs b/crates/ezd-ffmpeg/src/rate.rs new file mode 100644 index 0000000..f4e252c --- /dev/null +++ b/crates/ezd-ffmpeg/src/rate.rs @@ -0,0 +1,173 @@ +//! Rate-control numbers shared by every encoder context setup, and the report of what ran. + +use ezd_proto::{RateControl, RateMode}; + +/// The encoder may burst to this many times the negotiated bitrate. +pub const PEAK_FACTOR: u32 = 2; +/// VBV buffer size in frames at the peak rate. +pub const VBV_FRAMES: u32 = 4; +/// NVENC constant-quality target (`cq`). +pub const NVENC_CQ: u32 = 19; +/// libx264 constant rate factor. +pub const X264_CRF: u32 = 20; +/// Fixed quantiser (CRF for libx264) of the high-quality option sets. +pub const HIGH_QP: u32 = 10; +/// Fixed quantiser of the VAAPI CQP fallback sets. +pub const VAAPI_QP: u32 = 20; + +/// Values for `bit_rate`, `rc_max_rate` and `rc_buffer_size` of an encoder context. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RateSettings { + /// Target bitrate in bit/s. + pub bit_rate: i64, + /// Peak bitrate in bit/s. + pub max_rate: i64, + /// VBV buffer size in bits. + pub buffer_size: i32, +} + +/// Context rate settings for a negotiated bitrate and refresh rate (millihertz). +pub fn rate_settings(kbps: u32, refresh_mhz: u32) -> RateSettings { + let bit_rate = kbps as i64 * 1000; + let max_rate = bit_rate * PEAK_FACTOR as i64; + let buffer = max_rate * VBV_FRAMES as i64 * 1000 / refresh_mhz.max(1) as i64; + RateSettings { + bit_rate, + max_rate, + buffer_size: buffer.clamp(1000, i32::MAX as i64) as i32, + } +} + +/// Rate-control description of the encoder `name` opened with `options`, for the stream report. +pub fn describe_rate(name: &str, options: &[(&str, String)], kbps: u32) -> RateControl { + let get = |key: &str| { + options + .iter() + .find(|(k, _)| *k == key) + .map(|(_, v)| v.as_str()) + }; + let num = |key: &str| get(key).and_then(|v| v.parse::().ok()).unwrap_or(0); + let (mode, quality) = if get("tune") == Some("lossless") { + (RateMode::Lossless, 0) + } else if let Some(crf) = get("crf") { + (RateMode::Crf, crf.parse().unwrap_or(0)) + } else if let Some(rc) = get("rc") { + match rc { + "cbr" => (RateMode::Cbr, 0), + "vbr" | "vbr_peak" | "vbr_latency" => (RateMode::Vbr, num("cq")), + "cqp" | "constqp" => (RateMode::Cqp, num("qp").max(num("qp_i"))), + _ => (RateMode::Unknown, 0), + } + } else if let Some(rc) = get("rc_mode") { + match rc { + "CBR" => (RateMode::Cbr, 0), + "VBR" => (RateMode::Vbr, 0), + "CQP" => (RateMode::Cqp, num("qp")), + _ => (RateMode::Unknown, 0), + } + } else if name.ends_with("_qsv") { + // QSV runs VBR whenever the maximum rate is above the bitrate. + (RateMode::Vbr, 0) + } else { + (RateMode::Unknown, 0) + }; + RateControl { + mode: mode.code(), + quality, + target_kbps: kbps, + // A fixed quantiser and lossless ignore the rate limits. + max_kbps: if matches!(mode, RateMode::Cqp | RateMode::Lossless) { + 0 + } else { + kbps.saturating_mul(PEAK_FACTOR) + }, + } +} + +/// Compact `k=v` label of the interesting options of a set, joined by spaces. +pub fn options_label(options: &[(&str, String)]) -> String { + const KEYS: [&str; 9] = [ + "rc", + "rc_mode", + "cq", + "qp", + "qp_i", + "crf", + "low_power", + "profile", + "level", + ]; + options + .iter() + .filter(|(k, v)| KEYS.contains(k) || (*k == "tune" && v == "lossless")) + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(" ") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::option_sets; + use ezd_proto::QualityLevel; + + #[test] + fn settings_use_peak_and_four_frame_vbv() { + let r = rate_settings(45_949, 120_000); + assert_eq!(r.bit_rate, 45_949_000); + assert_eq!(r.max_rate, 91_898_000); + assert_eq!(r.buffer_size, (91_898_000i64 * 4 / 120) as i32); + assert!(rate_settings(1, 0).buffer_size >= 1000); + } + + #[test] + fn describes_the_set_that_opened() { + let d = |name: &str, i: usize| { + let sets = option_sets(name, QualityLevel::Normal); + describe_rate(name, &sets[i], 1000) + }; + let nv = d("h264_nvenc", 0); + assert_eq!((nv.rate_mode(), nv.quality), (RateMode::Vbr, 19)); + assert_eq!((nv.target_kbps, nv.max_kbps), (1000, 2000)); + let x = d("libx264", 0); + assert_eq!((x.rate_mode(), x.quality), (RateMode::Crf, 20)); + let va = d("h264_vaapi", 0); + assert_eq!((va.rate_mode(), va.quality), (RateMode::Vbr, 0)); + let cqp = d("h264_vaapi", 1); + assert_eq!((cqp.rate_mode(), cqp.quality), (RateMode::Cqp, 20)); + assert_eq!(cqp.max_kbps, 0); + assert_eq!(d("h264_qsv", 0).rate_mode(), RateMode::Vbr); + assert_eq!(d("libopenh264", 0).rate_mode(), RateMode::Unknown); + } + + #[test] + fn describes_high_and_lossless_sets() { + let sets = option_sets("hevc_nvenc", QualityLevel::Lossless); + let lossless = describe_rate("hevc_nvenc", &sets[0], 1000); + assert_eq!( + (lossless.rate_mode(), lossless.quality, lossless.max_kbps), + (RateMode::Lossless, 0, 0) + ); + assert_eq!(options_label(&sets[0]), "tune=lossless"); + let high = describe_rate("hevc_nvenc", &sets[1], 1000); + assert_eq!( + (high.rate_mode(), high.quality, high.max_kbps), + (RateMode::Cqp, 10, 0) + ); + let x = option_sets("libx264", QualityLevel::High); + let x = describe_rate("libx264", &x[0], 1000); + assert_eq!((x.rate_mode(), x.quality), (RateMode::Crf, 10)); + let amf = option_sets("hevc_amf", QualityLevel::High); + let amf = describe_rate("hevc_amf", &amf[0], 1000); + assert_eq!((amf.rate_mode(), amf.quality), (RateMode::Cqp, 10)); + } + + #[test] + fn label_keeps_only_interesting_keys() { + let sets = option_sets("h264_vaapi", QualityLevel::Normal); + assert_eq!( + options_label(&sets[1]), + "rc_mode=CQP qp=20 low_power=1 level=5.2" + ); + } +} diff --git a/crates/ezd-host-core/src/lib.rs b/crates/ezd-host-core/src/lib.rs index 0bce1d6..8217d55 100644 --- a/crates/ezd-host-core/src/lib.rs +++ b/crates/ezd-host-core/src/lib.rs @@ -28,7 +28,7 @@ pub use session::{ }; pub use source::{ AudioBackend, AudioBlock, AudioEncoder, AudioSource, CursorSink, DisplayMode, EncodedFrame, - OutputInfo, SourceError, SourceHdr, VideoSource, VirtualDisplay, + EncoderSetup, OutputInfo, SourceError, SourceHdr, VideoSource, VirtualDisplay, }; pub use synthetic::{PixelFormat, SyntheticFrames, read_frame_number}; pub use time::now_us; diff --git a/crates/ezd-host-core/src/session.rs b/crates/ezd-host-core/src/session.rs index ab01ae8..fe70038 100644 --- a/crates/ezd-host-core/src/session.rs +++ b/crates/ezd-host-core/src/session.rs @@ -7,6 +7,7 @@ //! |-- ClientCaps -------------->| //! |-- ClientHdr (minor >= 5) --->| the host waits for it when the client speaks minor >= 5 //! |-- ClientAudio (minor >= 7) ->| likewise; the host waits for it +//! |-- ClientPrefs (minor >= 8) ->| codec and quality the client asks for //! |<--------------- HostCaps ---| //! |<------------ StreamColor ---| minor >= 5 only //! |<------------ AudioConfig ---| minor >= 7 only; `stream: None` = no audio @@ -32,9 +33,10 @@ use crate::time::now_us; use ezd_link::{Link, MuxClosed, MuxError, MuxOptions, MuxReceiver, MuxSender, VideoCredit}; use ezd_proto::{ AudioCodec, AudioConfig, AudioPacket, AudioStream, AudioVolume, Bye, ClientAudio, ClientCaps, - ClientHdr, ClientOptions, ClientStats, ClockSync, Codec, DEFAULT_CHUNK_LEN, DynamicRange, - Hello, HostAudio, HostCaps, InputStatus, Message, Ping, Pong, Prefs, ProtoError, Reject, Role, - SampleFormat, StreamColor, StreamConfig, VideoFrame, negotiate_audio, negotiate_hdr, + ClientHdr, ClientOptions, ClientPrefs, ClientStats, ClockSync, Codec, DEFAULT_CHUNK_LEN, + DynamicRange, Hello, HostAudio, HostCaps, InputStatus, Message, Ping, Pong, Prefs, ProtoError, + QualityLevel, RateMode, Reject, Role, SampleFormat, StreamColor, StreamConfig, StreamReport, + VideoFrame, negotiate_audio, negotiate_hdr, }; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -322,6 +324,12 @@ pub async fn run_host_session_with( cfg, color, range: dynamic_range, + quality: + QualityPlan { + wanted: wanted_quality, + applied: applied_quality, + wanted_codec, + }, audio, } = match setup { Ok(v) => v, @@ -392,6 +400,16 @@ pub async fn run_host_session_with( { let (shared, tx, cfg) = (shared.clone(), tx.clone(), cfg.clone()); + let report_base = client_hello.supports_report().then(|| ReportBase { + wanted_kbps: prefs.bitrate_kbps.unwrap_or_else(|| { + ezd_proto::auto_bitrate_kbps(cfg.width, cfg.height, cfg.refresh_mhz, cfg.codec) + }), + user_bitrate: prefs.bitrate_kbps.is_some(), + ceiling_kbps: ezd_proto::bitrate_ceiling_kbps(&prefs), + wanted_quality, + applied_quality, + wanted_codec, + }); let input_for_video = input_tx.clone(); let handle = tokio::runtime::Handle::current(); let poll = opts.poll_interval; @@ -400,13 +418,14 @@ pub async fn run_host_session_with( .spawn(move || { let mut source = source; source.set_dynamic_range(dynamic_range, &color); + source.set_quality(applied_quality); let cursor = CursorSink::new(tx.clone()); let r = match source.start(&cfg, cursor) { Ok(()) => { if let Some(t) = &input_for_video { let _ = t.send(InputCmd::Output(source.output())); } - video_loop(source.as_mut(), &tx, &shared, &handle, poll) + video_loop(source.as_mut(), &tx, &shared, &handle, poll, report_base) } Err(e) => Err(e), }; @@ -811,9 +830,21 @@ struct Negotiated { cfg: StreamConfig, color: StreamColor, range: DynamicRange, + quality: QualityPlan, audio: Option, } +/// What the client asked for and what the host applies. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct QualityPlan { + /// Quality the client asked for (normal for a client without `ClientPrefs`). + wanted: QualityLevel, + /// Quality the host applies after its own rules. + applied: QualityLevel, + /// Codec the client prefers (`None` = auto). + wanted_codec: Option, +} + async fn handshake( tx: &MuxSender, rx: &mut MuxReceiver, @@ -868,11 +899,24 @@ async fn handshake( } else { None }; + // A client with minor >= 8 follows `ClientHdr` with its stream preferences. + let client_prefs: Option = if hello.supports_stream_prefs() { + Some( + expect(rx, tx, "ClientPrefs", wait, |m| match m { + Message::ClientPrefs(p) => Ok(p), + o => Err(o), + }) + .await?, + ) + } else { + None + }; tx.send_control(Message::HostCaps(host_caps.clone()))?; + let merged = prefs.with_client(client_prefs.as_ref()); let (cfg, range) = match negotiate_hdr( &caps, host_caps, - prefs, + &merged, hdr_in.source_hdr.is_some(), client_hdr.as_ref(), &hdr_in.main10, @@ -909,12 +953,18 @@ async fn handshake( o => Err(o), }) .await?; + let applied = merged.applied_quality(cfg.codec); Ok(Negotiated { hello, caps, cfg, color, range, + quality: QualityPlan { + wanted: client_prefs.map_or(QualityLevel::Normal, |p| p.quality()), + applied, + wanted_codec: client_prefs.and_then(|p| p.codec()), + }, audio, }) } @@ -963,12 +1013,23 @@ fn reject(tx: &MuxSender, reason: String) { let _ = tx.send_control(Message::Reject(Reject { reason })); } +/// Negotiation side of the `StreamReport`; the encoder side comes from the source. +struct ReportBase { + wanted_kbps: u32, + user_bitrate: bool, + ceiling_kbps: u32, + wanted_quality: QualityLevel, + applied_quality: QualityLevel, + wanted_codec: Option, +} + fn video_loop( source: &mut dyn VideoSource, tx: &MuxSender, sh: &VideoShared, handle: &tokio::runtime::Handle, poll: Duration, + report_base: Option, ) -> Result<(), SourceError> { let mut last_config: Option> = None; let mut next_id = 0u64; @@ -1005,6 +1066,48 @@ fn video_loop( let Some(f) = source.next_frame(poll)? else { continue; }; + // The encoder opened (again) to produce this frame: tell the client how, before it. + if let Some(setup) = source.take_encoder_setup() { + let report = StreamReport { + wanted_kbps: report_base.as_ref().map_or(0, |b| b.wanted_kbps), + user_bitrate: report_base.as_ref().is_some_and(|b| b.user_bitrate), + ceiling_kbps: report_base.as_ref().map_or(0, |b| b.ceiling_kbps), + attempts: setup.attempts, + rate: setup.rate, + wanted_quality: report_base.as_ref().map_or(0, |b| b.wanted_quality.code()), + applied_quality: report_base.as_ref().map_or(0, |b| { + // Lossless that did not open (the high sets ran) is reported as high. + match b.applied_quality { + QualityLevel::Lossless if setup.rate.rate_mode() != RateMode::Lossless => { + QualityLevel::High + } + q => q, + } + .code() + }), + wanted_codec: report_base + .as_ref() + .and_then(|b| b.wanted_codec) + .map_or(0, Codec::code), + }; + tracing::info!( + wanted_kbps = report.wanted_kbps, + user_bitrate = report.user_bitrate, + ceiling_kbps = report.ceiling_kbps, + rate_mode = report.rate.rate_mode().name(), + quality = report.rate.quality, + wanted_quality = report.wanted_quality, + applied_quality = report.applied_quality, + wanted_codec = report.wanted_codec, + target_kbps = report.rate.target_kbps, + max_kbps = report.rate.max_kbps, + attempts = ?report.attempts, + "encoder setup" + ); + if report_base.is_some() && tx.send_control(Message::StreamReport(report)).is_err() { + return Ok(()); + } + } let id = f.frame_id.unwrap_or(next_id); next_id = id + 1; if let Some(cfg) = f.codec_config @@ -1194,6 +1297,8 @@ mod tests { .unwrap(); tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); assert!(matches!(recv(&mut rx).await, Message::Hello(h) if h.role == Role::Host)); assert!(matches!(recv(&mut rx).await, Message::HostCaps(_))); assert_eq!( @@ -1343,6 +1448,8 @@ mod tests { .unwrap(); tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); while !matches!(recv(&mut rx).await, Message::StreamConfig(_)) {} tx.send_control(Message::ConfigAck).unwrap(); // The app restarts and opens the same link again. @@ -1383,6 +1490,8 @@ mod tests { .unwrap(); tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); assert!(matches!(recv(&mut rx).await, Message::Hello(_))); assert!(matches!(recv(&mut rx).await, Message::HostCaps(_))); assert!(matches!(recv(&mut rx).await, Message::Reject(_))); @@ -1475,6 +1584,10 @@ mod tests { tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); } + if minor >= ezd_proto::MINOR_STREAM_PREFS { + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); + } let mut seen = Vec::new(); loop { let m = recv(&mut rx).await; @@ -1495,6 +1608,212 @@ mod tests { (seen, told) } + /// A source that offers an encoder setup once, after its first frame. + struct ReportingSource { + inner: ScriptedSource, + pending: bool, + offered: bool, + quality: Arc>>, + } + + impl VideoSource for ReportingSource { + fn set_quality(&mut self, quality: QualityLevel) { + *self.quality.lock().unwrap() = Some(quality); + } + fn encoders(&self) -> Vec { + self.inner.encoders() + } + fn start(&mut self, cfg: &StreamConfig, cursor: CursorSink) -> Result<(), SourceError> { + self.inner.start(cfg, cursor) + } + fn next_frame(&mut self, t: Duration) -> Result, SourceError> { + let f = self.inner.next_frame(t); + self.pending = !std::mem::replace(&mut self.offered, true); + f + } + fn take_encoder_setup(&mut self) -> Option { + if !std::mem::take(&mut self.pending) { + return None; + } + Some(crate::source::EncoderSetup { + attempts: vec![ezd_proto::EncoderAttempt { + encoder: "enc".into(), + options: "rc=vbr".into(), + ok: true, + error: String::new(), + }], + rate: ezd_proto::RateControl { + mode: ezd_proto::RateMode::Vbr.code(), + quality: 19, + target_kbps: 1000, + max_kbps: 2000, + }, + }) + } + fn request_keyframe(&mut self) { + self.inner.request_keyframe() + } + fn stop(&mut self) { + self.inner.stop() + } + } + + /// Messages a client of `minor` receives up to its first few video frames. + async fn messages_until_video(minor: u16) -> Vec { + messages_with_prefs(minor, ClientPrefs::auto(), Prefs::default()) + .await + .0 + } + + /// Like [`messages_until_video`] with the client's preferences and the host's `Prefs`; also + /// returns the quality the source was told. + async fn messages_with_prefs( + minor: u16, + client_prefs: ClientPrefs, + prefs: Prefs, + ) -> (Vec, Option) { + let (host_link, client_link) = Link::duplex_pair(1 << 20); + let told = Arc::new(std::sync::Mutex::new(None)); + let src = ReportingSource { + inner: ScriptedSource::new(Duration::from_millis(5), 3000), + pending: false, + offered: false, + quality: told.clone(), + }; + let host = tokio::spawn(run_host_session_with( + host_link, + Box::new(src), + prefs, + fast_opts(), + )); + let (tx, mut rx, _t) = client_link.into_mux(DEFAULT_CHUNK_LEN); + let mut hello = Hello::new(Role::Client, "t", "client"); + hello.proto_minor = minor; + tx.send_control(Message::Hello(hello)).unwrap(); + tx.send_control(Message::ClientCaps(caps())).unwrap(); + tx.send_control(Message::ClientHdr(ClientHdr::none())) + .unwrap(); + if minor >= ezd_proto::MINOR_AUDIO { + tx.send_control(Message::ClientAudio(ClientAudio::none())) + .unwrap(); + } + if minor >= ezd_proto::MINOR_STREAM_PREFS { + tx.send_control(Message::ClientPrefs(client_prefs)).unwrap(); + } + loop { + if matches!(recv(&mut rx).await, Message::StreamConfig(_)) { + break; + } + } + tx.send_control(Message::ConfigAck).unwrap(); + let mut seen = Vec::new(); + let mut videos = 0; + while videos < 5 { + let m = recv(&mut rx).await; + videos += matches!(m, Message::VideoFrame(_)) as usize; + seen.push(m); + } + tx.send_control(Message::Bye(Bye { reason: "t".into() })) + .unwrap(); + let _ = timeout(Duration::from_secs(10), host).await; + let told = *told.lock().unwrap(); + (seen, told) + } + + #[tokio::test] + async fn report_precedes_the_first_frame_for_a_new_client() { + let seen = messages_until_video(ezd_proto::MINOR_STREAM_PREFS).await; + let report = seen + .iter() + .position(|m| matches!(m, Message::StreamReport(_))) + .expect("no StreamReport"); + let first_video = seen + .iter() + .position(|m| matches!(m, Message::VideoFrame(_))) + .unwrap(); + assert!(report < first_video, "{seen:?}"); + let Message::StreamReport(r) = &seen[report] else { + unreachable!() + }; + assert!(!r.user_bitrate && r.wanted_kbps > 0 && r.ceiling_kbps >= r.wanted_kbps); + assert_eq!(r.attempts.len(), 1); + assert_eq!(r.rate.quality, 19); + let count = seen + .iter() + .filter(|m| matches!(m, Message::StreamReport(_))) + .count(); + assert_eq!(count, 1, "one setup, one report"); + } + + fn report_of(seen: &[Message]) -> &StreamReport { + seen.iter() + .find_map(|m| match m { + Message::StreamReport(r) => Some(r), + _ => None, + }) + .expect("no StreamReport") + } + + #[tokio::test] + async fn client_quality_reaches_the_source_and_the_report() { + let high = ClientPrefs::new(Some(Codec::H264), QualityLevel::High); + let (seen, told) = + messages_with_prefs(ezd_proto::PROTOCOL_MINOR, high, Prefs::default()).await; + assert_eq!(told, Some(QualityLevel::High)); + let r = report_of(&seen); + assert_eq!( + (r.wanted_quality, r.applied_quality, r.wanted_codec), + (QualityLevel::High.code(), QualityLevel::High.code(), 1) + ); + } + + #[tokio::test] + async fn network_link_and_h264_downgrade_the_applied_quality() { + let lossless = ClientPrefs::new(None, QualityLevel::Lossless); + // The scripted client decodes H.264 only: lossless falls back to high. + let (seen, told) = + messages_with_prefs(ezd_proto::PROTOCOL_MINOR, lossless, Prefs::default()).await; + assert_eq!(told, Some(QualityLevel::High)); + let r = report_of(&seen); + assert_eq!( + (r.wanted_quality, r.applied_quality), + (QualityLevel::Lossless.code(), QualityLevel::High.code()) + ); + let net = Prefs { + link: ezd_proto::LinkKind::Network, + ..Prefs::default() + }; + let (seen, told) = messages_with_prefs(ezd_proto::PROTOCOL_MINOR, lossless, net).await; + assert_eq!(told, Some(QualityLevel::Normal)); + let r = report_of(&seen); + assert_eq!( + (r.wanted_quality, r.applied_quality), + (QualityLevel::Lossless.code(), QualityLevel::Normal.code()) + ); + } + + #[tokio::test] + async fn host_quality_overrides_the_client() { + let host = Prefs { + quality: QualityLevel::High, + ..Prefs::default() + }; + let (seen, told) = + messages_with_prefs(ezd_proto::PROTOCOL_MINOR, ClientPrefs::auto(), host).await; + assert_eq!(told, Some(QualityLevel::High)); + let r = report_of(&seen); + assert_eq!( + (r.wanted_quality, r.applied_quality), + (QualityLevel::Normal.code(), QualityLevel::High.code()) + ); + } + + #[tokio::test] + async fn old_client_gets_no_report() { + let seen = messages_until_video(ezd_proto::MINOR_STREAM_PREFS - 1).await; + assert!(!seen.iter().any(|m| matches!(m, Message::StreamReport(_)))); + } + #[tokio::test] async fn hdr10_is_negotiated_with_a_capable_client() { let hdr = ClientHdr::new(true, 1000, 600, 50, &[Codec::Hevc]); @@ -1596,6 +1915,8 @@ mod tests { .unwrap(); tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); for _ in 0..4 { recv(&mut rx).await; } @@ -1642,6 +1963,8 @@ mod tests { .unwrap(); tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); for _ in 0..4 { recv(&mut rx).await; } @@ -1687,6 +2010,8 @@ mod tests { .unwrap(); tx.send_control(Message::ClientAudio(ClientAudio::none())) .unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); for _ in 0..4 { recv(&mut rx).await; } @@ -1811,6 +2136,10 @@ mod tests { if let Some(a) = client_audio { tx.send_control(Message::ClientAudio(a)).unwrap(); } + if minor >= ezd_proto::MINOR_STREAM_PREFS { + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); + } let mut seen = Vec::new(); loop { let m = recv(&mut rx).await; @@ -1885,6 +2214,8 @@ mod tests { tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); tx.send_control(Message::ClientAudio(pcm_client())).unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); loop { if matches!(recv(&mut rx).await, Message::StreamConfig(_)) { break; diff --git a/crates/ezd-host-core/src/source.rs b/crates/ezd-host-core/src/source.rs index 0892a04..855e8a6 100644 --- a/crates/ezd-host-core/src/source.rs +++ b/crates/ezd-host-core/src/source.rs @@ -2,8 +2,8 @@ use ezd_link::{MuxClosed, MuxSender}; use ezd_proto::{ - AudioVolume, Codec, CursorShape, CursorState, DynamicRange, EncoderCaps, Mastering, - StreamColor, StreamConfig, + AudioVolume, Codec, CursorShape, CursorState, DynamicRange, EncoderAttempt, EncoderCaps, + Mastering, QualityLevel, RateControl, StreamColor, StreamConfig, }; use std::collections::HashSet; use std::sync::{Arc, Mutex}; @@ -139,6 +139,15 @@ impl SourceHdr { } } +/// What a source's encoder tried and what it runs with, for the `StreamReport` sent to the client. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncoderSetup { + /// Encoders and option sets tried, in order. + pub attempts: Vec, + /// Rate control of the encoder that opened. + pub rate: RateControl, +} + /// A capture + encode pipeline for one virtual display. /// /// Implemented per platform (and by `ezd_ffmpeg::SwVideoSource` for tests). @@ -172,6 +181,10 @@ pub trait VideoSource: Send { /// ignored (SDR BT.709 stream). fn set_dynamic_range(&mut self, _range: DynamicRange, _color: &StreamColor) {} + /// Tells the source the quality level the host applies, right before + /// [`start`](Self::start). Default: ignored (the normal, rate-controlled stream). + fn set_quality(&mut self, _quality: QualityLevel) {} + /// Starts capture and encoding with the negotiated configuration. The /// first produced frame must be a keyframe carrying `codec_config`. /// `cursor` stays valid until `stop`. @@ -192,6 +205,13 @@ pub trait VideoSource: Send { None } + /// The setup of the encoder that opened since the last call, `None` when nothing changed + /// (the default for sources that do not report). The session polls it after every frame + /// from `next_frame` and sends it to the client before that frame. + fn take_encoder_setup(&mut self) -> Option { + None + } + /// Makes the next frame returned by `next_frame` a keyframe. fn request_keyframe(&mut self); diff --git a/crates/ezd-host-linux/src/ahead.rs b/crates/ezd-host-linux/src/ahead.rs index 8c51cb1..da1c0eb 100644 --- a/crates/ezd-host-linux/src/ahead.rs +++ b/crates/ezd-host-linux/src/ahead.rs @@ -281,6 +281,7 @@ mod tests { name: "libx264", kind: Kind::X264, node: None, + ten_bit: false, } } diff --git a/crates/ezd-host-linux/src/diag.rs b/crates/ezd-host-linux/src/diag.rs index acbf974..bdf5e15 100644 --- a/crates/ezd-host-linux/src/diag.rs +++ b/crates/ezd-host-linux/src/diag.rs @@ -26,9 +26,9 @@ pub fn section() -> Value { .map(|n| { let mut encs = Vec::new(); let plans = [if n.is_nvidia() { - Plan { name: "h264_nvenc", kind: Kind::Nvenc, node: Some(n.path.clone()) } + Plan { name: "h264_nvenc", kind: Kind::Nvenc, node: Some(n.path.clone()), ten_bit: false } } else { - Plan { name: "h264_vaapi", kind: Kind::Vaapi, node: Some(n.path.clone()) } + Plan { name: "h264_vaapi", kind: Kind::Vaapi, node: Some(n.path.clone()), ten_bit: false } }]; // Only non-NVIDIA nodes are probed in diag so the dGPU is not woken up needlessly. for p in plans { @@ -56,6 +56,7 @@ pub fn section() -> Value { name: "libx264", kind: Kind::X264, node: None, + ten_bit: false, }; let sw = plan_works(&x264); let token = |k| portal::token_path(k).is_some_and(|p| p.exists()); diff --git a/crates/ezd-host-linux/src/encode.rs b/crates/ezd-host-linux/src/encode.rs index 3043011..45f4cc2 100644 --- a/crates/ezd-host-linux/src/encode.rs +++ b/crates/ezd-host-linux/src/encode.rs @@ -10,7 +10,8 @@ use ezd_ffmpeg::ColorInfo; use ezd_ffmpeg::HwDevice; use ezd_ffmpeg::HwKind; use ezd_ffmpeg::sys as ff; -use ezd_host_core::now_us; +use ezd_host_core::{EncoderSetup, now_us}; +use ezd_proto::{Codec, EncoderAttempt, QualityLevel}; use std::ffi::{CStr, CString, c_void}; use std::os::fd::IntoRawFd; use std::ptr; @@ -36,6 +37,8 @@ pub struct Plan { pub kind: Kind, /// Render node used (VAAPI, NVENC for description). pub node: Option, + /// 10-bit HDR10 variant (HEVC Main10, P010 input) instead of the 8-bit SDR one. + pub ten_bit: bool, } impl Plan { @@ -44,15 +47,19 @@ impl Plan { self.kind != Kind::X264 } - /// The HDR10 variant: HEVC Main10 (P010) instead of 8-bit H.264. + /// The HDR10 variant: HEVC Main10 (P010) instead of the 8-bit SDR one. pub fn hdr10(&self) -> bool { - ten_bit_encoder(self.name) + self.ten_bit } -} -/// Encoders that only run the 10-bit HDR10 chain (there is no 8-bit HEVC plan). -fn ten_bit_encoder(name: &str) -> bool { - name.starts_with("hevc_") + /// The codec this plan produces. + pub fn codec(&self) -> Codec { + if self.name.starts_with("hevc_") { + Codec::Hevc + } else { + Codec::H264 + } + } } /// Chooses the HDR10 encoders for `node`: HEVC Main10, best first. Empty without a hardware @@ -67,43 +74,47 @@ pub fn plan_hdr_encoders(node: Option<&RenderNode>, works: &dyn Fn(&Plan) -> boo name: "hevc_nvenc", kind: Kind::Nvenc, node: Some(n.path.clone()), + ten_bit: true, } } else { Plan { name: "hevc_vaapi", kind: Kind::Vaapi, node: Some(n.path.clone()), + ten_bit: true, } }; if works(&plan) { vec![plan] } else { Vec::new() } } -/// Chooses the encoders to offer for `node`, best first. `works` tells whether a plan was -/// verified on this machine. libx264 is always the last resort. +/// Chooses the SDR encoders to offer for `node`, best first: the H.264 primary, then 8-bit HEVC +/// on the same node (the tablet can ask for it, and lossless needs it). `works` tells whether a +/// plan was verified on this machine. libx264 is always the last resort. pub fn plan_encoders(node: Option<&RenderNode>, works: &dyn Fn(&Plan) -> bool) -> Vec { let mut out = Vec::new(); if let Some(n) = node { - let primary = if n.is_nvidia() { - Plan { - name: "h264_nvenc", - kind: Kind::Nvenc, - node: Some(n.path.clone()), - } + let (primary, hevc) = if n.is_nvidia() { + (("h264_nvenc", Kind::Nvenc), ("hevc_nvenc", Kind::Nvenc)) } else { - Plan { - name: "h264_vaapi", - kind: Kind::Vaapi, + (("h264_vaapi", Kind::Vaapi), ("hevc_vaapi", Kind::Vaapi)) + }; + for (name, kind) in [primary, hevc] { + let plan = Plan { + name, + kind, node: Some(n.path.clone()), + ten_bit: false, + }; + if works(&plan) { + out.push(plan); } - }; - if works(&primary) { - out.push(primary); } } let x264 = Plan { name: "libx264", kind: Kind::X264, node: None, + ten_bit: false, }; if works(&x264) { out.push(x264); @@ -345,6 +356,16 @@ struct Enc { ctx: *mut ff::AVCodecContext, pkt: *mut ff::AVPacket, extradata: Vec, + /// What `open` tried and what it runs with; taken by the `Encoder` once. + setup: Option, +} + +/// Longest error text kept in an [`EncoderAttempt`]. +const ATTEMPT_ERROR_MAX: usize = 120; + +/// `text` cut to what fits an [`EncoderAttempt`] error. +pub(crate) fn short_error(text: &str) -> String { + text.chars().take(ATTEMPT_ERROR_MAX).collect() } // SAFETY: exclusively owned FFmpeg objects, moved between threads only as a whole. @@ -372,6 +393,8 @@ struct EncParams<'a> { color: ColorInfo, /// 10-bit input (HEVC Main10 option sets). ten_bit: bool, + /// Quality level selecting the option sets. + quality: QualityLevel, } /// The FFmpeg enums of the H.273 codes of `c`; codes we do not stream stay unspecified. @@ -421,11 +444,28 @@ impl Enc { return Err(format!("encoder {} not available in this FFmpeg", p.name)); } let mut last = String::from("no option set"); - for set in ezd_ffmpeg::option_sets_for(p.name, p.ten_bit) { + let mut attempts = Vec::new(); + for set in ezd_ffmpeg::option_sets_for(p.name, p.ten_bit, p.quality) { + let mut attempt = EncoderAttempt { + encoder: p.name.to_string(), + options: ezd_ffmpeg::options_label(&set), + ok: true, + error: String::new(), + }; match Self::try_open(codec, p, &set) { - Ok(e) => return Ok(e), + Ok(mut e) => { + attempts.push(attempt); + e.setup = Some(EncoderSetup { + attempts, + rate: ezd_ffmpeg::describe_rate(p.name, &set, p.kbps), + }); + return Ok(e); + } Err(e) => { tracing::debug!("opening {} with {set:?} failed: {e}", p.name); + attempt.ok = false; + attempt.error = short_error(&e); + attempts.push(attempt); last = e; } } @@ -449,9 +489,8 @@ impl Enc { ctx, pkt: ptr::null_mut(), extradata: Vec::new(), + setup: None, }; - let bitrate = p.kbps as i64 * 1000; - let fps = p.fps_mhz as f64 / 1000.0; let c = &mut *ctx; c.width = p.width as i32; c.height = p.height as i32; @@ -464,9 +503,10 @@ impl Enc { den: 1000, }; c.pix_fmt = p.pix_fmt; - c.bit_rate = bitrate; - c.rc_max_rate = bitrate; - c.rc_buffer_size = ((bitrate as f64 * 2.0 / fps) as i32).max(1000); + let rate = ezd_ffmpeg::rate_settings(p.kbps, p.fps_mhz); + c.bit_rate = rate.bit_rate; + c.rc_max_rate = rate.max_rate; + c.rc_buffer_size = rate.buffer_size; c.gop_size = i32::MAX; c.max_b_frames = 0; c.thread_count = 1; @@ -1169,10 +1209,34 @@ impl Converter { struct Active { generation: u64, + /// Encodes every captured frame (the lossless level uses its high set here). enc: Enc, sent_config: bool, + /// Lossless level only: the lossless encoder that re-encodes a still image. + refine: Option, } +/// Progressive refinement: the lossless encoder and the frame waiting for it. +struct Refine { + enc: Enc, + /// The last packet sent to the client came from `enc`, so the decoder's references are the + /// refined picture and the next motion frame must be an IDR. + last_sent: bool, + pending: Option, +} + +/// The last frame the motion encoder took, kept for a lossless re-encode. +struct Pending { + frame: OwnedFrame, + path: &'static str, + /// Keeps the capture buffer valid until the refine is done. + _release: Option, +} + +// SAFETY: the frame is exclusively owned (a clone with its own reference counts, which are +// atomic) and moves between threads as a whole, like `Prepared`. +unsafe impl Send for Pending {} + /// Encodes [`Prepared`] frames. A new encoder is opened when the converter reopened; its first /// packet is an IDR with codec config. pub struct Encoder { @@ -1180,7 +1244,12 @@ pub struct Encoder { tags: StreamTags, fps_mhz: u32, kbps: u32, + /// 10-bit HDR10 stream (HEVC Main10). + ten_bit: bool, + quality: QualityLevel, active: Option, + /// Setup of the encoder opened last, until the source takes it. + setup: Option, } impl Encoder { @@ -1191,10 +1260,30 @@ impl Encoder { tags: StreamTags::default(), fps_mhz, kbps, + ten_bit: false, + quality: QualityLevel::Normal, active: None, + setup: None, } } + /// Makes this the 10-bit HDR10 encoder (HEVC Main10) of `plan`. + pub fn with_ten_bit(mut self, ten_bit: bool) -> Encoder { + self.ten_bit = ten_bit; + self + } + + /// Sets the quality level that selects the option sets (default normal). + pub fn with_quality(mut self, quality: QualityLevel) -> Encoder { + self.quality = quality; + self + } + + /// The setup of the encoder opened since the last call (every open makes a new one). + pub fn take_setup(&mut self) -> Option { + self.setup.take() + } + /// Sets the colour signalling of the stream (default BT.709 limited range). pub fn with_tags(mut self, tags: StreamTags) -> Encoder { self.tags = tags; @@ -1212,7 +1301,7 @@ impl Encoder { self.active = None; // SAFETY: valid frame; the frames context outlives the call. let hw = unsafe { (*p.frame.0).hw_frames_ctx }; - let enc = Enc::open(&EncParams { + let params = |quality| EncParams { name: self.name, width: p.width, height: p.height, @@ -1221,16 +1310,47 @@ impl Encoder { pix_fmt: p.pix_fmt, hw_frames: (!hw.is_null()).then_some(hw), color: self.tags.color, - ten_bit: ten_bit_encoder(self.name), - })?; + ten_bit: self.ten_bit, + quality, + }; + let mut enc = Enc::open(¶ms(self.quality))?; + let mut setup = enc.setup.take(); + let mut refine = None; + let lossless_ok = self.quality == QualityLevel::Lossless + && setup + .as_ref() + .is_some_and(|s| s.rate.rate_mode() == ezd_proto::RateMode::Lossless); + let mut motion = enc; + if lossless_ok { + match Enc::open(¶ms(QualityLevel::High)) { + Ok(mut high) => { + if let (Some(s), Some(h)) = (setup.as_mut(), high.setup.take()) { + s.attempts.extend(h.attempts); + } + // The high set encodes every frame, the lossless one refines. + refine = Some(Refine { + enc: std::mem::replace(&mut motion, high), + last_sent: false, + pending: None, + }); + } + Err(e) => tracing::warn!( + "high set for progressive refinement did not open ({e}); \ + encoding lossless only" + ), + } + } + self.setup = setup; self.active = Some(Active { generation: p.generation, - enc, + enc: motion, sent_config: false, + refine, }); } let a = self.active.as_mut().expect("opened above"); - let want_key = force_key || !a.sent_config; + let want_key = + force_key || !a.sent_config || a.refine.as_ref().is_some_and(|r| r.last_sent); let Prepared { frame, path, @@ -1244,7 +1364,7 @@ impl Encoder { // SAFETY: `frame` is a valid frame we own. unsafe { let f = &mut *frame.0; - if ten_bit_encoder(self.name) { + if self.ten_bit { // HDR10 frames carry the stream's colour tags (8-bit streams keep what the // conversion left, which the encoder ignores in favour of the context). (f.color_primaries, f.color_trc, f.colorspace, f.color_range) = @@ -1258,6 +1378,15 @@ impl Encoder { f.flags &= !(ff::AV_FRAME_FLAG_KEY); } } + // Refine mode keeps the frame and the capture buffer for a lossless re-encode. + let keep = match a.refine.is_some() { + // SAFETY: `frame` is valid; the clone shares its buffers. + true => Some(OwnedFrame(unsafe { ff::av_frame_clone(frame.0) })), + false => None, + }; + if keep.as_ref().is_some_and(|k| k.0.is_null()) { + return Err("av_frame_clone failed".into()); + } let t_send = now_us(); a.enc.send(frame.0)?; drop(frame); @@ -1273,7 +1402,19 @@ impl Encoder { receive_us = encoded_us - t_receive, "encode stages" ); - // The GPU/CPU is done with the capture buffer once the packet is out. + // The GPU/CPU is done with the capture buffer once the packet is out, unless a refine + // still needs it. + let release = match (a.refine.as_mut(), keep) { + (Some(r), Some(frame)) => { + r.pending = Some(Pending { + frame, + path, + _release: release, + }); + None + } + _ => release, + }; drop(release); let Some((data, keyframe)) = packet else { return Ok(None); @@ -1284,6 +1425,9 @@ impl Encoder { None }; a.sent_config = true; + if let Some(r) = a.refine.as_mut() { + r.last_sent = false; + } Ok(Some(Output { data, keyframe, @@ -1292,6 +1436,60 @@ impl Encoder { path, })) } + + /// Whether a frame waits for its lossless re-encode ([`Encoder::refine`]). + pub fn refine_pending(&self) -> bool { + self.active + .as_ref() + .and_then(|a| a.refine.as_ref()) + .is_some_and(|r| r.pending.is_some()) + } + + /// Re-encodes the last frame with the lossless encoder as one IDR with codec config. A frame + /// is refined once; `None` when nothing is pending. A failing refine encoder is dropped and + /// the stream goes on with motion frames only. + pub fn refine(&mut self) -> Result, String> { + let Some(a) = self.active.as_mut() else { + return Ok(None); + }; + let Some(r) = a.refine.as_mut() else { + return Ok(None); + }; + let Some(pending) = r.pending.take() else { + return Ok(None); + }; + let Pending { frame, path, .. } = pending; + // SAFETY: `frame` is a valid frame we own. + unsafe { + let f = &mut *frame.0; + f.pict_type = ff::AVPictureType::AV_PICTURE_TYPE_I; + f.flags |= ff::AV_FRAME_FLAG_KEY; + } + let result = r + .enc + .send(frame.0) + .and_then(|()| r.enc.receive()) + .map(|p| (p, now_us())); + drop(frame); + match result { + Ok((Some((data, keyframe)), encoded_us)) => { + r.last_sent = true; + Ok(Some(Output { + data, + keyframe, + codec_config: Some(r.enc.extradata.clone()), + encoded_us, + path, + })) + } + Ok((None, _)) => Ok(None), + Err(e) => { + tracing::warn!("lossless refinement failed ({e}); disabled for this stream"); + a.refine = None; + Ok(None) + } + } + } } /// Converter and encoder in series on the calling thread. @@ -1304,11 +1502,34 @@ impl Pipeline { /// New pipeline; nothing is opened until the first frame arrives. pub fn new(plan: Plan, fps_mhz: u32, kbps: u32) -> Pipeline { Pipeline { - enc: Encoder::new(plan.name, fps_mhz, kbps), + enc: Encoder::new(plan.name, fps_mhz, kbps).with_ten_bit(plan.ten_bit), conv: Converter::new(plan, fps_mhz), } } + /// Sets the quality level that selects the encoder option sets (default normal). + pub fn with_quality(self, quality: QualityLevel) -> Pipeline { + Pipeline { + enc: self.enc.with_quality(quality), + conv: self.conv, + } + } + + /// The setup of the encoder opened since the last call. + pub fn take_setup(&mut self) -> Option { + self.enc.take_setup() + } + + /// Whether a frame waits for its lossless re-encode. + pub fn refine_pending(&self) -> bool { + self.enc.refine_pending() + } + + /// Re-encodes the last frame losslessly, see [`Encoder::refine`]. + pub fn refine(&mut self) -> Result, String> { + self.enc.refine() + } + /// Sets the colour signalling of the stream (default BT.709 limited range). pub fn with_tags(self, tags: StreamTags) -> Pipeline { Pipeline { @@ -1862,12 +2083,29 @@ mod tests { let intel = plan_encoders(Some(&node("/dev/dri/renderD129", 0x8086)), &all); assert_eq!( intel.iter().map(|p| p.name).collect::>(), - ["h264_vaapi", "libx264"] + ["h264_vaapi", "hevc_vaapi", "libx264"] ); assert_eq!(intel[0].node.as_deref(), Some("/dev/dri/renderD129")); - assert!(intel[0].hardware() && !intel[1].hardware()); + assert!(intel[0].hardware() && intel[1].hardware() && !intel[2].hardware()); + // The 8-bit HEVC plan is SDR: 8-bit input, no HDR10 chain. + assert!(!intel[1].hdr10() && !intel[1].ten_bit); + assert_eq!( + intel.iter().map(Plan::codec).collect::>(), + [Codec::H264, Codec::Hevc, Codec::H264] + ); let nv = plan_encoders(Some(&node("/dev/dri/renderD128", 0x10de)), &all); - assert_eq!(nv[0].name, "h264_nvenc"); + assert_eq!( + nv.iter().map(|p| p.name).collect::>(), + ["h264_nvenc", "hevc_nvenc", "libx264"] + ); + // The HEVC plan is offered only when it was verified on this machine. + let no_hevc = plan_encoders(Some(&node("/dev/dri/renderD128", 0x10de)), &|p| { + !p.name.starts_with("hevc_") + }); + assert_eq!( + no_hevc.iter().map(|p| p.name).collect::>(), + ["h264_nvenc", "libx264"] + ); // A broken hardware path leaves only x264; nothing works -> empty. let sw = plan_encoders(Some(&node("/dev/dri/renderD129", 0x8086)), &|p| { p.kind == Kind::X264 @@ -1919,6 +2157,7 @@ mod tests { name: "libx264", kind: Kind::X264, node: None, + ten_bit: false, }; let mut p = Pipeline::new(plan, 60_000, 8_000); let out = p @@ -1948,6 +2187,7 @@ mod tests { name: "libx264", kind: Kind::X264, node: None, + ten_bit: false, }; let mut p = Pipeline::new(plan, 60_000, 8_000); p.encode(test_shm_frame(320, 240, Q), false) @@ -1978,6 +2218,7 @@ mod tests { name: "h264_vaapi", kind: Kind::Vaapi, node: Some(node), + ten_bit: false, }; let mut p = Pipeline::new(plan, 60_000, 8_000); match p.encode(test_shm_frame(640, 360, Q), false) { @@ -1996,6 +2237,7 @@ mod tests { name: "hevc_vaapi", kind: Kind::Vaapi, node: Some(node), + ten_bit: true, } } @@ -2284,6 +2526,7 @@ mod tests { name: "h264_vaapi", kind: Kind::Vaapi, node: Some(node), + ten_bit: false, }; let mut p = Pipeline::new(plan, 60_000, 8_000); let out = p @@ -2312,6 +2555,7 @@ mod tests { name: "h264_vaapi", kind: Kind::Vaapi, node: Some(node), + ten_bit: false, }; let mut conv = Converter::new(plan, 60_000); let p = conv @@ -2504,11 +2748,143 @@ mod tests { ); } + fn hevc_nvenc_sdr_plan(node: String) -> Plan { + Plan { + name: "hevc_nvenc", + kind: Kind::Nvenc, + node: Some(node), + ten_bit: false, + } + } + + fn decode_hevc_first(out: &Output) -> ezd_ffmpeg::DecodedFrame { + let mut d = Decoder::new(Codec::Hevc).unwrap(); + if let Some(c) = &out.codec_config { + d.push_config(c).unwrap(); + } + let mut frames = d.decode(&out.data, 0).unwrap(); + frames.extend(d.flush().unwrap()); + frames.into_iter().next().expect("decoded a picture") + } + + /// 8-bit HEVC on NVENC from system memory at the tablet's size: the high set (constant + /// quantiser) and the lossless tune open, report what ran, and decode with the right colours. + #[test] + fn nvenc_hevc_8bit_high_and_lossless_if_available() { + let Some(node) = crate::adapter::list_render_nodes() + .into_iter() + .find(|n| n.is_nvidia()) + else { + return eprintln!("skip: no NVIDIA render node"); + }; + for (quality, mode, first_ok) in [ + (QualityLevel::High, ezd_proto::RateMode::Cqp, true), + (QualityLevel::Lossless, ezd_proto::RateMode::Lossless, true), + ] { + let mut p = Pipeline::new(hevc_nvenc_sdr_plan(node.path.clone()), 60_000, 8_000) + .with_quality(quality); + let out = match p.encode(test_shm_frame(2960, 1848, Q), false) { + Ok(Some(o)) => o, + Ok(None) => panic!("{quality:?}: no packet"), + Err(e) => return eprintln!("skip {quality:?}: {e}"), + }; + let setup = p.take_setup().expect("setup after the open"); + assert_eq!(setup.attempts[0].ok, first_ok, "{quality:?}: {setup:?}"); + assert_eq!(setup.rate.rate_mode(), mode, "{quality:?}: {setup:?}"); + if quality == QualityLevel::High { + assert_eq!(setup.rate.quality, ezd_ffmpeg::HIGH_QP as u16); + } else { + assert!(setup.attempts[0].options.contains("tune=lossless")); + } + assert!(out.keyframe && out.codec_config.is_some()); + check_colors(&decode_hevc_first(&out)); + } + } + + /// The lossless level encodes moving frames with the high set and refines the last one with the + /// lossless set; the other levels never have anything to refine. + #[test] + fn nvenc_hevc_lossless_refines_the_last_frame_if_available() { + let Some(node) = crate::adapter::list_render_nodes() + .into_iter() + .find(|n| n.is_nvidia()) + else { + return eprintln!("skip: no NVIDIA render node"); + }; + let plan = || hevc_nvenc_sdr_plan(node.path.clone()); + let mut p = Pipeline::new(plan(), 60_000, 8_000).with_quality(QualityLevel::Lossless); + let first = match p.encode(test_shm_frame(2960, 1848, Q), false) { + Ok(Some(o)) => o, + Ok(None) => panic!("no packet"), + Err(e) => return eprintln!("skip: {e}"), + }; + assert!(first.keyframe && first.codec_config.is_some()); + let setup = p.take_setup().expect("setup after the open"); + assert_eq!(setup.rate.rate_mode(), ezd_proto::RateMode::Lossless); + let options: Vec<&str> = setup.attempts.iter().map(|a| a.options.as_str()).collect(); + assert!( + options.iter().any(|o| o.contains("tune=lossless")), + "{options:?}" + ); + assert!(options.iter().any(|o| o.contains("constqp")), "{options:?}"); + assert!(p.refine_pending()); + let refined = p.refine().unwrap().expect("refined frame"); + assert!(refined.keyframe && refined.codec_config.is_some()); + assert_ne!(refined.data, first.data); + check_colors(&decode_hevc_first(&refined)); + // A frame is refined once. + assert!(!p.refine_pending()); + assert!(p.refine().unwrap().is_none()); + // The decoder's references were replaced: the next motion frame is an IDR with config. + let second = p + .encode(test_shm_frame(2960, 1848, Q), false) + .unwrap() + .unwrap(); + assert!(second.keyframe && second.codec_config.is_some()); + let third = p + .encode(test_shm_frame(2960, 1848, Q), false) + .unwrap() + .unwrap(); + assert!(!third.keyframe && third.codec_config.is_none()); + assert!(p.refine_pending()); + for quality in [QualityLevel::Normal, QualityLevel::High] { + let mut p = Pipeline::new(plan(), 60_000, 8_000).with_quality(quality); + p.encode(test_shm_frame(2960, 1848, Q), false) + .unwrap() + .unwrap(); + assert!(!p.refine_pending(), "{quality:?}"); + assert!(p.refine().unwrap().is_none(), "{quality:?}"); + } + } + + /// libx264 at the high level runs CRF 10 and reports it. + #[test] + fn x264_high_quality_reports_crf() { + let plan = Plan { + name: "libx264", + kind: Kind::X264, + node: None, + ten_bit: false, + }; + let mut p = Pipeline::new(plan, 60_000, 8_000).with_quality(QualityLevel::High); + let out = p + .encode(test_shm_frame(320, 240, Q), false) + .unwrap() + .unwrap(); + let setup = p.take_setup().expect("setup after the open"); + assert_eq!( + (setup.rate.rate_mode(), setup.rate.quality), + (ezd_proto::RateMode::Crf, 10) + ); + check_colors(&decode_first(&out)); + } + fn nvenc_plan(node: String) -> Plan { Plan { name: "h264_nvenc", kind: Kind::Nvenc, node: Some(node), + ten_bit: false, } } @@ -2570,10 +2946,25 @@ mod tests { assert!(p.release.is_some()); let out = std::thread::spawn(move || { let mut enc = Encoder::new("h264_nvenc", 60_000, 8_000); - enc.encode(p, false).expect("encode").expect("packet") + let out = enc.encode(p, false).expect("encode").expect("packet"); + (out, enc.take_setup(), enc.take_setup()) }) .join() .unwrap(); + let (out, setup, again) = out; + // The open made one setup: constant quality inside the peak, taken once. + let setup = setup.expect("setup after the open"); + assert!(again.is_none()); + assert_eq!(setup.attempts.len(), 1); + assert!(setup.attempts[0].ok && setup.attempts[0].options.contains("rc=vbr cq=19")); + assert_eq!( + (setup.rate.mode, setup.rate.quality), + (ezd_proto::RateMode::Vbr.code(), 19) + ); + assert_eq!( + (setup.rate.target_kbps, setup.rate.max_kbps), + (8_000, 16_000) + ); assert!(out.keyframe && out.codec_config.is_some()); check_colors(&decode_first(&out)); } @@ -2654,6 +3045,7 @@ mod tests { name: "h264_vaapi", kind: Kind::Vaapi, node: Some(node.clone()), + ten_bit: false, }; bench("vaapi-shm", plan.clone(), &|| test_shm_frame(1920, 1080, Q)); bench("vaapi-dmabuf", plan, &|| { @@ -2664,6 +3056,7 @@ mod tests { name: "libx264", kind: Kind::X264, node: None, + ten_bit: false, }; bench("x264", x, &|| test_shm_frame(1920, 1080, Q)); } diff --git a/crates/ezd-host-linux/src/kscreen.rs b/crates/ezd-host-linux/src/kscreen.rs index 47a31ae..6af5738 100644 --- a/crates/ezd-host-linux/src/kscreen.rs +++ b/crates/ezd-host-linux/src/kscreen.rs @@ -160,11 +160,14 @@ pub fn find_mode(out: &KOutput, width: u32, height: u32, refresh_mhz: u32) -> Op } /// Argument for `kscreen-doctor` that adds a reduced-blanking custom mode. +/// +/// `output` must be the numeric id: kscreen-doctor splits arguments on dots, and portal virtual +/// outputs are named after the caller's app id (`Virtual-virtual-xdp-kde-org.example.App`). pub fn add_custom_mode_arg(output: &str, width: u32, height: u32, refresh_mhz: u32) -> String { format!("output.{output}.addCustomMode.{width}.{height}.{refresh_mhz}.reduced") } -/// Argument for `kscreen-doctor` that switches `output` to mode `mode` (id or `WxH@R`). +/// Argument for `kscreen-doctor` that switches `output` (numeric id) to mode `mode` (id or `WxH@R`). pub fn set_mode_arg(output: &str, mode: &str) -> String { format!("output.{output}.mode.{mode}") } @@ -225,7 +228,12 @@ pub fn apply_mode( let (mut out, mut mode) = find(&list_outputs()?).ok_or_else(|| format!("output {output} not found"))?; if mode.is_none() { - run(&[add_custom_mode_arg(output, width, height, refresh_mhz)])?; + run(&[add_custom_mode_arg( + &out.id.to_string(), + width, + height, + refresh_mhz, + )])?; // KWin applies the new mode list asynchronously. for _ in 0..10 { std::thread::sleep(Duration::from_millis(100)); @@ -238,19 +246,28 @@ pub fn apply_mode( } } } + let id = out.id.to_string(); let target = mode.unwrap_or_else(|| mode_name(width, height, refresh_mhz)); if out.current_mode_id != target { - run(&[set_mode_arg(output, &target)])?; + run(&[set_mode_arg(&id, &target)])?; } for _ in 0..10 { if let Some(o) = list_outputs()?.into_iter().find(|o| o.name == output) - && o.size == (width, height) + && runs_mode(&o, width, height, refresh_mhz) { return Ok(o); } std::thread::sleep(Duration::from_millis(100)); } - Err(format!("{output} did not switch to {width}x{height}")) + Err(format!( + "{output} did not switch to {}", + mode_name(width, height, refresh_mhz) + )) +} + +/// True if the current mode of `out` is `width`x`height` at `refresh_mhz`. +fn runs_mode(out: &KOutput, width: u32, height: u32, refresh_mhz: u32) -> bool { + find_mode(out, width, height, refresh_mhz).is_some_and(|m| m.id == out.current_mode_id) } #[cfg(test)] @@ -351,6 +368,16 @@ mod tests { assert_eq!(mode_name(2560, 1600, 119_880), "2560x1600@120"); } + #[test] + fn runs_mode_checks_the_refresh_rate() { + let mut v = parse_outputs(&with_virtual()).unwrap().remove(1); + v.current_mode_id = "m1".into(); + assert!(runs_mode(&v, 2560, 1600, 120_000)); + assert!(!runs_mode(&v, 2560, 1600, 60_000)); + v.current_mode_id = "other".into(); + assert!(!runs_mode(&v, 2560, 1600, 120_000)); + } + #[test] fn scale_defaults_to_one() { let outs = parse_outputs(&with_virtual()).unwrap(); diff --git a/crates/ezd-host-linux/src/source.rs b/crates/ezd-host-linux/src/source.rs index 66f74a5..f0aebea 100644 --- a/crates/ezd-host-linux/src/source.rs +++ b/crates/ezd-host-linux/src/source.rs @@ -5,7 +5,7 @@ use crate::adapter::{self, RenderNode}; use crate::ahead::{ConvertAhead, Item, Start}; use crate::encode::{ Converter, Encoder, Kind, Pipeline, Plan, StreamTags, plan_encoders, plan_hdr_encoders, - probe_plan, + probe_plan, short_error, }; use crate::frame::{Frame, NegotiatedFormat}; use crate::portal::SourceKind; @@ -14,9 +14,12 @@ use crate::pw::{Frames, PwCapture, PwParams, default_intel_modifiers, default_nv use crate::wl_capture::WlCapture; use crate::x11_capture::X11Capture; use ezd_host_core::{ - CursorSink, EncodedFrame, OutputInfo, PlatformOptions, SourceError, SourceHdr, VideoSource, + CursorSink, EncodedFrame, EncoderSetup, OutputInfo, PlatformOptions, SourceError, SourceHdr, + VideoSource, +}; +use ezd_proto::{ + Codec, DynamicRange, EncoderAttempt, EncoderCaps, QualityLevel, StreamColor, StreamConfig, }; -use ezd_proto::{Codec, DynamicRange, EncoderCaps, StreamColor, StreamConfig}; use std::collections::HashMap; use std::sync::{Mutex, OnceLock}; use std::time::{Duration, Instant}; @@ -30,7 +33,11 @@ fn cache() -> &'static ProbeCache { /// Probes `plan` once per process (the result is cached, including failures). pub fn plan_works(plan: &Plan) -> Result<(), String> { - let key = (plan.name.to_string(), plan.node.clone().unwrap_or_default()); + // The 8-bit and the Main10 plan of an HEVC encoder are probed separately. + let key = ( + format!("{}{}", plan.name, if plan.ten_bit { "/10" } else { "" }), + plan.node.clone().unwrap_or_default(), + ); if let Some(r) = cache().lock().unwrap_or_else(|e| e.into_inner()).get(&key) { return r.clone(); } @@ -86,10 +93,14 @@ pub struct LinuxSource { /// Dynamic range negotiated for the session and the matching stream colour. range: DynamicRange, color: StreamColor, + /// Quality level the host applies (see `set_quality`). + quality: QualityLevel, running: Option, force_key: bool, /// The output the stream shows (virtual output or the captured monitor), after `start`. output: Option, + /// Encoders the probes found unusable, reported with the encoder setup. + probe_failures: Vec, } /// Converts and encodes capture frames. @@ -170,6 +181,21 @@ struct Running { dmabuf_ok: bool, /// When the shared-memory fallback started. dmabuf_off_at: Option, + /// The last frame encoded: when, and its capture time and counter (a refine reuses them). + last_encoded: Instant, + last_ids: (u64, u64), +} + +/// How long the image must stay unchanged before the lossless level re-encodes the last frame +/// losslessly. Moving content is encoded with the much smaller high set, so the stream keeps its +/// frame rate; the lossless frame is large and slow to encode, send and decode, so it is only +/// worth it once nothing else is waiting. Short enough to look immediate once the motion stops. +const REFINE_DELAY: Duration = Duration::from_millis(150); + +/// How long `next_frame` may wait for a frame while a refine is pending: until the image has been +/// still for [`REFINE_DELAY`] (`since_last` after the last encoded frame), but never past `timeout`. +fn refine_wait(timeout: Duration, since_last: Duration) -> Duration { + timeout.min(REFINE_DELAY.saturating_sub(since_last)) } /// After the shared-memory fallback starts, DMA-BUF frames that were already captured or @@ -202,14 +228,29 @@ impl LinuxSource { tracing::warn!("no DRM render node found; software encoding only"); } let node = node.map(|s| s.node); - let plans = plan_encoders(node.as_ref(), &|p| plan_works(p).is_ok()); + // Encoders that were probed unusable, in plan order, for the stream report. + let probe_failures = std::cell::RefCell::new(Vec::new()); + let works = |p: &Plan| match plan_works(p) { + Ok(()) => true, + Err(e) => { + probe_failures.borrow_mut().push(EncoderAttempt { + encoder: p.name.to_string(), + options: "probe".into(), + ok: false, + error: short_error(&e), + }); + false + } + }; + let plans = plan_encoders(node.as_ref(), &works); let hdr_source = observed_hdr(); // Probing opens a Main10 encoder: only worth it when the compositor offers HDR10. let hdr_plans = if hdr_source.is_some() { - plan_hdr_encoders(node.as_ref(), &|p| plan_works(p).is_ok()) + plan_hdr_encoders(node.as_ref(), &works) } else { Vec::new() }; + let probe_failures = probe_failures.into_inner(); LinuxSource { provider, kind: if opts.output_name.is_some() { @@ -224,9 +265,11 @@ impl LinuxSource { hdr_source, range: DynamicRange::Sdr, color: StreamColor::sdr_bt709(), + quality: QualityLevel::Normal, running: None, force_key: false, output: None, + probe_failures, } } @@ -248,18 +291,18 @@ impl LinuxSource { impl VideoSource for LinuxSource { fn encoders(&self) -> Vec { - let codec = |p: &Plan| { - if p.hdr10() { Codec::Hevc } else { Codec::H264 } - }; - self.plans - .iter() - .chain(&self.hdr_plans) - .map(|p| EncoderCaps { - codec: codec(p), - name: p.name.to_string(), - hardware: p.hardware(), - }) - .collect() + // The SDR and the Main10 plan of an HEVC encoder share a name: offer it once. + let mut caps: Vec = Vec::new(); + for p in self.plans.iter().chain(&self.hdr_plans) { + if !caps.iter().any(|c| c.name == p.name) { + caps.push(EncoderCaps { + codec: p.codec(), + name: p.name.to_string(), + hardware: p.hardware(), + }); + } + } + caps } fn source_hdr(&self) -> Option { @@ -279,6 +322,10 @@ impl VideoSource for LinuxSource { self.color = *color; } + fn set_quality(&mut self, quality: QualityLevel) { + self.quality = quality; + } + fn gpu_description(&self) -> String { self.node .as_ref() @@ -287,20 +334,13 @@ impl VideoSource for LinuxSource { } fn start(&mut self, cfg: &StreamConfig, cursor: CursorSink) -> Result<(), SourceError> { - // H.264 is the SDR codec; HEVC only exists here as HDR10 (Main10). + // H.264 and HEVC stream as SDR 8-bit; HEVC is also the HDR10 codec (Main10). let hdr10 = matches!(self.range, DynamicRange::Hdr10 { codec } if codec == cfg.codec); - match (cfg.codec, hdr10) { - (Codec::H264, false) | (Codec::Hevc, true) => {} - (Codec::Hevc, false) => { - return Err(SourceError::msg( - "the Linux host only produces HEVC for HDR10 streams", - )); - } - (codec, _) => { - return Err(SourceError::msg(format!( - "the Linux host cannot encode {codec:?}" - ))); - } + if !matches!(cfg.codec, Codec::H264 | Codec::Hevc) { + return Err(SourceError::msg(format!( + "the Linux host cannot encode {:?}", + cfg.codec + ))); } let plan = self .plans @@ -374,7 +414,10 @@ impl VideoSource for LinuxSource { tracing::info!(convert_ahead = ?ahead, "encode pipeline ready"); let input = match ahead { Some(start) => Input::Ahead { - enc: Encoder::new(plan.name, cfg.refresh_mhz, cfg.bitrate_kbps).with_tags(tags), + enc: Encoder::new(plan.name, cfg.refresh_mhz, cfg.bitrate_kbps) + .with_ten_bit(plan.ten_bit) + .with_quality(self.quality) + .with_tags(tags), conv: ConvertAhead::start( Converter::new(plan, cfg.refresh_mhz).with_tags(tags), capture.frames(), @@ -383,7 +426,9 @@ impl VideoSource for LinuxSource { .map_err(SourceError::msg)?, }, None => Input::Serial( - Pipeline::new(plan, cfg.refresh_mhz, cfg.bitrate_kbps).with_tags(tags), + Pipeline::new(plan, cfg.refresh_mhz, cfg.bitrate_kbps) + .with_quality(self.quality) + .with_tags(tags), ), }; self.running = Some(Running { @@ -392,6 +437,8 @@ impl VideoSource for LinuxSource { _output: opened.guard, dmabuf_ok: fmt.modifier.is_some(), dmabuf_off_at: None, + last_encoded: Instant::now(), + last_ids: (0, 0), }); self.force_key = true; Ok(()) @@ -403,44 +450,76 @@ impl VideoSource for LinuxSource { return Err(SourceError::msg("source not started")); }; // `start_us`: when this frame's conversion (serial) or encoding (ahead) began. + let mut refined = false; let (result, start_us, capture_us, seq, was_dma) = match &mut r.input { Input::Serial(pipeline) => { - let frame = match r.capture.next(timeout) { - Ok(Some(f)) => f, - Ok(None) => { + let pending = pipeline.refine_pending(); + let wait = if pending { + refine_wait(timeout, r.last_encoded.elapsed()) + } else { + timeout + }; + let frame = match r.capture.next(wait) { + Ok(f) => f, + Err(e) => return Err(SourceError::msg(e)), + }; + match frame { + Some(frame) => { + // Benchmark aid (tools/kwin-anim-bench): drop every frame to measure KWin + // alone. + if std::env::var_os("EZD_DIAG_SKIP_ENCODE").is_some() { + drop(frame); + return Ok(None); + } + let (capture_us, seq, dma) = (frame.capture_us, frame.seq, frame.is_dma()); + let start_us = ezd_host_core::now_us(); + let result = pipeline.encode(frame, force); + r.last_encoded = Instant::now(); + r.last_ids = (capture_us, seq); + (result, start_us, capture_us, seq, dma) + } + None if pending && r.last_encoded.elapsed() >= REFINE_DELAY => { + refined = true; + let start_us = ezd_host_core::now_us(); + let (capture_us, seq) = r.last_ids; + (pipeline.refine(), start_us, capture_us, seq, false) + } + None => { self.force_key |= force; return Ok(None); } - Err(e) => return Err(SourceError::msg(e)), - }; - // Benchmark aid (tools/kwin-anim-bench): drop every frame to measure KWin alone. - if std::env::var_os("EZD_DIAG_SKIP_ENCODE").is_some() { - drop(frame); - return Ok(None); } - let (capture_us, seq, dma) = (frame.capture_us, frame.seq, frame.is_dma()); - let start_us = ezd_host_core::now_us(); - ( - pipeline.encode(frame, force), - start_us, - capture_us, - seq, - dma, - ) } - Input::Ahead { conv, enc } => match conv.take(timeout) { - None => { - self.force_key |= force; - return Ok(None); - } - Some(Item::Frame(p)) => { - let (capture_us, seq, dma) = (p.capture_us, p.seq, p.dma); - let start_us = ezd_host_core::now_us(); - (enc.encode(p, force), start_us, capture_us, seq, dma) + Input::Ahead { conv, enc } => { + let pending = enc.refine_pending(); + let wait = if pending { + refine_wait(timeout, r.last_encoded.elapsed()) + } else { + timeout + }; + match conv.take(wait) { + None if pending && r.last_encoded.elapsed() >= REFINE_DELAY => { + refined = true; + let start_us = ezd_host_core::now_us(); + let (capture_us, seq) = r.last_ids; + (enc.refine(), start_us, capture_us, seq, false) + } + None => { + self.force_key |= force; + return Ok(None); + } + Some(Item::Frame(p)) => { + let (capture_us, seq, dma) = (p.capture_us, p.seq, p.dma); + let start_us = ezd_host_core::now_us(); + let result = enc.encode(p, force); + r.last_encoded = Instant::now(); + r.last_ids = (capture_us, seq); + (result, start_us, capture_us, seq, dma) + } + Some(Item::Failed { dma, error }) => (Err(error), 0, 0, 0, dma), + Some(Item::Closed(e)) => return Err(SourceError::msg(e)), } - Some(Item::Failed { dma, error }) => (Err(error), 0, 0, 0, dma), - Some(Item::Closed(e)) => return Err(SourceError::msg(e)), - }, + } }; match result { Ok(Some(o)) => { @@ -448,6 +527,7 @@ impl VideoSource for LinuxSource { path = o.path, bytes = o.data.len(), key = o.keyframe, + refine = refined, age_us = start_us.saturating_sub(capture_us), encode_us = o.encoded_us.saturating_sub(start_us), "encoded" @@ -486,6 +566,19 @@ impl VideoSource for LinuxSource { self.output.clone() } + fn take_encoder_setup(&mut self) -> Option { + let r = self.running.as_mut()?; + let mut setup = match &mut r.input { + Input::Serial(pipeline) => pipeline.take_setup(), + Input::Ahead { enc, .. } => enc.take_setup(), + }?; + // Probe failures happened before any open: they come first. + let mut attempts = self.probe_failures.clone(); + attempts.append(&mut setup.attempts); + setup.attempts = attempts; + Some(setup) + } + fn request_keyframe(&mut self) { self.force_key = true; } @@ -505,6 +598,19 @@ impl VideoSource for LinuxSource { mod tests { use super::*; + #[test] + fn refine_wait_runs_to_the_still_deadline_but_not_past_the_timeout() { + let ms = Duration::from_millis; + // Just encoded: wait the whole delay, if the caller allows it. + assert_eq!(refine_wait(ms(500), ms(0)), REFINE_DELAY); + assert_eq!(refine_wait(ms(100), ms(0)), ms(100)); + // Part of the delay has passed. + assert_eq!(refine_wait(ms(500), ms(100)), ms(50)); + // Overdue: do not wait. + assert_eq!(refine_wait(ms(500), ms(150)), ms(0)); + assert_eq!(refine_wait(ms(500), ms(900)), ms(0)); + } + #[test] fn hdr_state_changes_end_the_session_for_a_new_negotiation() { // Today's compositors: 8-bit sRGB all the time, nothing to renegotiate. diff --git a/crates/ezd-host-service/src/config.rs b/crates/ezd-host-service/src/config.rs index 1433abb..1af3829 100644 --- a/crates/ezd-host-service/src/config.rs +++ b/crates/ezd-host-service/src/config.rs @@ -5,7 +5,7 @@ use ezd_host_core::{ UnavailableBackend, }; use ezd_link::VideoCredit; -use ezd_proto::{AudioMode, HdrMode, LinkKind, Prefs}; +use ezd_proto::{AudioMode, Codec, HdrMode, LinkKind, Prefs, QualityLevel}; use serde::{Deserialize, Serialize}; use std::net::SocketAddr; use std::sync::Arc; @@ -95,6 +95,55 @@ impl From for HdrMode { } } +/// Video codec the host asks for (serializable mirror of `Option`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum CodecChoice { + /// Use the tablet's preference, else the best codec both sides support. + #[default] + Auto, + /// H.264. + H264, + /// HEVC. + Hevc, + /// AV1. + Av1, +} + +impl From for Option { + fn from(c: CodecChoice) -> Self { + match c { + CodecChoice::Auto => None, + CodecChoice::H264 => Some(Codec::H264), + CodecChoice::Hevc => Some(Codec::Hevc), + CodecChoice::Av1 => Some(Codec::Av1), + } + } +} + +/// Stream quality the host asks for (serializable mirror of [`QualityLevel`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum QualityChoice { + /// Use the tablet's choice (rate-controlled stream if it makes none). + #[default] + Normal, + /// Near-transparent fixed quantiser (USB only), whatever the tablet asks. + High, + /// Lossless HEVC (USB only), whatever the tablet asks. + Lossless, +} + +impl From for QualityLevel { + fn from(c: QualityChoice) -> Self { + match c { + QualityChoice::Normal => QualityLevel::Normal, + QualityChoice::High => QualityLevel::High, + QualityChoice::Lossless => QualityLevel::Lossless, + } + } +} + /// PC audio streaming policy (serializable mirror of [`AudioMode`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] @@ -216,6 +265,11 @@ pub struct ServiceConfig { pub hdr: HdrChoice, /// PC audio policy (`audio = "auto" | "pcm" | "opus" | "off"`). pub audio: AudioChoice, + /// Codec to use, overriding the tablet's preference; `auto` follows the tablet. + pub codec: CodecChoice, + /// Quality to use, overriding the tablet's choice unless `normal`. Above `normal` it applies + /// to USB sessions only. + pub quality: QualityChoice, /// Wi-Fi listener; `None` leaves Wi-Fi off. Sessions on it always use the network link kind, /// whatever `link` says. #[cfg(feature = "wifi")] @@ -247,6 +301,8 @@ impl Default for ServiceConfig { open_app_on_connect: true, hdr: HdrChoice::default(), audio: AudioChoice::default(), + codec: CodecChoice::default(), + quality: QualityChoice::default(), #[cfg(feature = "wifi")] wifi: None, detach_grace_ms: ezd_host_core::DEFAULT_DETACH_GRACE.as_millis() as u32, @@ -268,6 +324,8 @@ impl ServiceConfig { scale_permille: self.scale_permille, hdr: self.hdr.into(), audio: self.audio.into(), + codec: self.codec.into(), + quality: self.quality.into(), ..Prefs::default() } } @@ -348,6 +406,8 @@ mod tests { encoder: Some("h264_nvenc".into()), scale_permille: 750, hdr: HdrChoice::Off, + codec: CodecChoice::Hevc, + quality: QualityChoice::High, ..ServiceConfig::default() }; assert_eq!( @@ -359,6 +419,8 @@ mod tests { encoder: Some("h264_nvenc".into()), scale_permille: 750, hdr: HdrMode::Off, + codec: Some(Codec::Hevc), + quality: QualityLevel::High, ..Prefs::default() } ); diff --git a/crates/ezd-host-service/src/lib.rs b/crates/ezd-host-service/src/lib.rs index 5f23b4a..901ec77 100644 --- a/crates/ezd-host-service/src/lib.rs +++ b/crates/ezd-host-service/src/lib.rs @@ -32,7 +32,8 @@ mod wifi; pub use adb::{AdbClient, AdbLocator, is_network_serial}; pub use config::{ - AdbMode, AoaMode, AudioChoice, HdrChoice, LinkChoice, PenChoice, ServiceConfig, SourceKind, + AdbMode, AoaMode, AudioChoice, CodecChoice, HdrChoice, LinkChoice, PenChoice, QualityChoice, + ServiceConfig, SourceKind, }; #[cfg(feature = "wifi")] pub use config::{CongestionChoice, WifiConfig}; diff --git a/crates/ezd-host-service/src/service.rs b/crates/ezd-host-service/src/service.rs index 7082323..518b753 100644 --- a/crates/ezd-host-service/src/service.rs +++ b/crates/ezd-host-service/src/service.rs @@ -653,8 +653,8 @@ pub(crate) mod tests { use ezd_link::adb::DeviceState as AdbState; use ezd_link::{MuxReceiver, MuxSender}; use ezd_proto::{ - Bye, ClientAudio, ClientCaps, ClientHdr, Codec, DEFAULT_CHUNK_LEN, DecoderCaps, - DisplayInfo, Hello, Message, Role, + Bye, ClientAudio, ClientCaps, ClientHdr, ClientPrefs, Codec, DEFAULT_CHUNK_LEN, + DecoderCaps, DisplayInfo, Hello, Message, Role, }; use tokio::time::timeout; @@ -745,6 +745,8 @@ pub(crate) mod tests { tx.send_control(Message::ClientHdr(ClientHdr::none())) .unwrap(); tx.send_control(Message::ClientAudio(audio)).unwrap(); + tx.send_control(Message::ClientPrefs(ClientPrefs::auto())) + .unwrap(); let mut seen = 0; let mut audio_stream = None; while seen < 5 { diff --git a/crates/ezd-host-windows/src/win/ffenc.rs b/crates/ezd-host-windows/src/win/ffenc.rs index d7ec178..e37c26a 100644 --- a/crates/ezd-host-windows/src/win/ffenc.rs +++ b/crates/ezd-host-windows/src/win/ffenc.rs @@ -351,7 +351,10 @@ impl ZeroCopyEncoder { return Err(format!("encoder {name} not available in this FFmpeg build")); } let mut last = String::from("no option set"); - for set in ezd_ffmpeg::option_sets_for(name, format.ten_bit) { + // The Windows host does not plumb the quality level yet: always the normal sets. + for set in + ezd_ffmpeg::option_sets_for(name, format.ten_bit, ezd_proto::QualityLevel::Normal) + { match enc.try_open(codec, &set, width, height, refresh_mhz, bitrate_kbps) { Ok(()) => { enc.pkt = ffi::av_packet_alloc(); @@ -388,8 +391,6 @@ impl ZeroCopyEncoder { return Err("avcodec_alloc_context3 failed".into()); } self.ctx = ctx; - let bitrate = bitrate_kbps as i64 * 1000; - let fps = refresh_mhz as f64 / 1000.0; let c = &mut *ctx; c.width = width as i32; c.height = height as i32; @@ -402,9 +403,10 @@ impl ZeroCopyEncoder { den: 1000, }; c.pix_fmt = ffi::AVPixelFormat::AV_PIX_FMT_D3D11; - c.bit_rate = bitrate; - c.rc_max_rate = bitrate; - c.rc_buffer_size = ((bitrate as f64 * 2.0 / fps) as i32).max(1000); + let rate = ezd_ffmpeg::rate_settings(bitrate_kbps, refresh_mhz); + c.bit_rate = rate.bit_rate; + c.rc_max_rate = rate.max_rate; + c.rc_buffer_size = rate.buffer_size; c.gop_size = i32::MAX; c.max_b_frames = 0; c.thread_count = 1; diff --git a/crates/ezd-host/i18n/de/ezd-host.ftl b/crates/ezd-host/i18n/de/ezd-host.ftl index 16fad28..8e9a790 100644 --- a/crates/ezd-host/i18n/de/ezd-host.ftl +++ b/crates/ezd-host/i18n/de/ezd-host.ftl @@ -15,6 +15,8 @@ run-link = Verbindungsart: wählt die Standard-Bitratenobergrenze (usb 80 Mbit/s run-max-refresh-mhz = Höchste zu verwendende Bildwiederholrate in Millihertz. run-bitrate-kbps = Feste Bitrate in kbit/s (auf die Obergrenze der Verbindung begrenzt). run-encoder = Erzwingt einen bestimmten Encoder per Namen. +run-codec = Videocodec: auto, h264, hevc oder av1. Auto folgt der Wahl des Tablets, sonst dem besten Codec, den beide Seiten unterstützen. +run-quality = Streamqualität: normal, high oder lossless. High und lossless gelten nur bei einer USB-Verbindung (lossless benötigt HEVC, sonst high). Normal folgt der Wahl des Tablets. run-adapter = Zu verwendende GPU/Adapter (`--source virtual`). run-prefer-dgpu = Bevorzugt die dedizierte GPU (`--source virtual`). run-output = Nimmt diese vorhandene Ausgabe auf, statt eine neue zu erstellen (`--source virtual`). diff --git a/crates/ezd-host/i18n/en/ezd-host.ftl b/crates/ezd-host/i18n/en/ezd-host.ftl index a1d12f5..1f1b492 100644 --- a/crates/ezd-host/i18n/en/ezd-host.ftl +++ b/crates/ezd-host/i18n/en/ezd-host.ftl @@ -15,6 +15,8 @@ run-link = Link kind: selects the default bitrate ceiling (usb 80 Mbps, network run-max-refresh-mhz = Highest refresh rate to use, in millihertz. run-bitrate-kbps = Fixed bitrate in kbit/s (clamped to the link ceiling). run-encoder = Force a specific encoder by name. +run-codec = Video codec: auto, h264, hevc or av1. Auto follows the tablet's choice, else the best codec both sides support. +run-quality = Stream quality: normal, high or lossless. High and lossless apply on a USB link only (lossless needs HEVC, else high). Normal follows the tablet's choice. run-adapter = GPU/adapter to use (`--source virtual`). run-prefer-dgpu = Prefer the discrete GPU (`--source virtual`). run-output = Capture this existing output instead of creating one (`--source virtual`). diff --git a/crates/ezd-host/i18n/es/ezd-host.ftl b/crates/ezd-host/i18n/es/ezd-host.ftl index 7a55408..4f8d10a 100644 --- a/crates/ezd-host/i18n/es/ezd-host.ftl +++ b/crates/ezd-host/i18n/es/ezd-host.ftl @@ -15,6 +15,8 @@ run-link = Tipo de enlace: elige el límite de bitrate por defecto (usb 80 Mbps, run-max-refresh-mhz = Frecuencia de actualización máxima a usar, en milihercios. run-bitrate-kbps = Bitrate fijo en kbit/s (limitado al máximo del enlace). run-encoder = Fuerza un codificador concreto por nombre. +run-codec = Códec de video: auto, h264, hevc o av1. Auto sigue la elección de la tablet y, si no, el mejor códec que soporten ambos. +run-quality = Calidad del stream: normal, high o lossless. High y lossless se aplican solo en un enlace USB (lossless requiere HEVC; si no, high). Normal sigue la elección de la tablet. run-adapter = GPU/adaptador a usar (`--source virtual`). run-prefer-dgpu = Prefiere la GPU dedicada (`--source virtual`). run-output = Captura esta salida existente en lugar de crear una (`--source virtual`). diff --git a/crates/ezd-host/i18n/fr/ezd-host.ftl b/crates/ezd-host/i18n/fr/ezd-host.ftl index 32d741c..cf9ec9d 100644 --- a/crates/ezd-host/i18n/fr/ezd-host.ftl +++ b/crates/ezd-host/i18n/fr/ezd-host.ftl @@ -15,6 +15,8 @@ run-link = Type de liaison : choisit le débit maximal par défaut (usb 80 Mbit/ run-max-refresh-mhz = Fréquence de rafraîchissement maximale à utiliser, en millihertz. run-bitrate-kbps = Débit fixe en kbit/s (limité au maximum de la liaison). run-encoder = Force un encodeur précis par son nom. +run-codec = Codec vidéo : auto, h264, hevc ou av1. Auto suit le choix de la tablette, sinon le meilleur codec pris en charge des deux côtés. +run-quality = Qualité du flux : normal, high ou lossless. High et lossless ne s'appliquent que sur une liaison USB (lossless nécessite HEVC, sinon high). Normal suit le choix de la tablette. run-adapter = GPU/adaptateur à utiliser (`--source virtual`). run-prefer-dgpu = Préfère le GPU dédié (`--source virtual`). run-output = Capture cette sortie existante au lieu d'en créer une (`--source virtual`). diff --git a/crates/ezd-host/i18n/it/ezd-host.ftl b/crates/ezd-host/i18n/it/ezd-host.ftl index a38845a..d65fae9 100644 --- a/crates/ezd-host/i18n/it/ezd-host.ftl +++ b/crates/ezd-host/i18n/it/ezd-host.ftl @@ -15,6 +15,8 @@ run-link = Tipo di collegamento: sceglie il limite di bitrate predefinito (usb 8 run-max-refresh-mhz = Frequenza di aggiornamento massima da usare, in millihertz. run-bitrate-kbps = Bitrate fisso in kbit/s (limitato al massimo del collegamento). run-encoder = Forza un encoder specifico per nome. +run-codec = Codec video: auto, h264, hevc o av1. Auto segue la scelta del tablet, altrimenti il miglior codec supportato da entrambi. +run-quality = Qualità dello stream: normal, high o lossless. High e lossless valgono solo su un collegamento USB (lossless richiede HEVC, altrimenti high). Normal segue la scelta del tablet. run-adapter = GPU/adattatore da usare (`--source virtual`). run-prefer-dgpu = Preferisce la GPU dedicata (`--source virtual`). run-output = Cattura questa uscita esistente invece di crearne una (`--source virtual`). diff --git a/crates/ezd-host/i18n/pt/ezd-host.ftl b/crates/ezd-host/i18n/pt/ezd-host.ftl index 8c6e4b4..a551b21 100644 --- a/crates/ezd-host/i18n/pt/ezd-host.ftl +++ b/crates/ezd-host/i18n/pt/ezd-host.ftl @@ -15,6 +15,8 @@ run-link = Tipo de conexão: define o limite de bitrate padrão (usb 80 Mbps, re run-max-refresh-mhz = Maior taxa de atualização a usar, em milihertz. run-bitrate-kbps = Bitrate fixo em kbit/s (limitado ao máximo da conexão). run-encoder = Força um codificador específico pelo nome. +run-codec = Codec de vídeo: auto, h264, hevc ou av1. Auto segue a escolha do tablet e, senão, o melhor codec suportado pelos dois lados. +run-quality = Qualidade do stream: normal, high ou lossless. High e lossless valem apenas em um link USB (lossless requer HEVC, senão high). Normal segue a escolha do tablet. run-adapter = GPU/adaptador a usar (`--source virtual`). run-prefer-dgpu = Prefere a GPU dedicada (`--source virtual`). run-output = Captura esta saída existente em vez de criar uma (`--source virtual`). diff --git a/crates/ezd-host/src/run.rs b/crates/ezd-host/src/run.rs index c9ef1ac..040a9ea 100644 --- a/crates/ezd-host/src/run.rs +++ b/crates/ezd-host/src/run.rs @@ -5,8 +5,8 @@ use anyhow::{Result, anyhow}; use clap::{Args, Command, ValueEnum}; use ezd_host_core::SessionSummary; use ezd_host_service::{ - AdbMode, AoaMode, AudioChoice, HostEvent, HostService, HostStatus, LinkChoice, PenChoice, - ServiceConfig, ServiceError, SourceKind, + AdbMode, AoaMode, AudioChoice, CodecChoice, HostEvent, HostService, HostStatus, LinkChoice, + PenChoice, QualityChoice, ServiceConfig, ServiceError, SourceKind, }; use i18n_embed::fluent::FluentLanguageLoader; use std::net::ToSocketAddrs; @@ -43,6 +43,21 @@ pub enum PenModeArg { Mouse, } +#[derive(Debug, Clone, Copy, ValueEnum)] +pub enum CodecArg { + Auto, + H264, + Hevc, + Av1, +} + +#[derive(Debug, Clone, Copy, ValueEnum)] +pub enum QualityArg { + Normal, + High, + Lossless, +} + // Option help texts live in i18n//ezd-host.ftl (`run-