diff --git a/CMakeLists.txt b/CMakeLists.txt index ef3469a..ad789bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE BOOL "Export compile commands for IDE set(MOONLIGHT_BUILD_KIND "HOST" CACHE STRING "Internal Moonlight build mode") set_property(CACHE MOONLIGHT_BUILD_KIND PROPERTY STRINGS HOST XBOX) option(MOONLIGHT_BUILD_VIDEO_EXPERIMENTS "Build standalone GPU presentation and MPEG-2 sampling experiments" OFF) -option(MOONLIGHT_ENABLE_GPU_PRESENTATION "Enable experimental NV2A overlay presentation before software fallback" ON) option(MOONLIGHT_INPUT_DIAGNOSTICS "Display digital controller state while diagnosing stream input" OFF) if(MOONLIGHT_BUILD_KIND STREQUAL "XBOX") diff --git a/README.md b/README.md index c49fad9..470749e 100644 --- a/README.md +++ b/README.md @@ -72,8 +72,16 @@ Only the required SVGs and their license are packaged on the ISO. No network connection or JavaScript runtime is needed to display the icons on the console. +## Video presentation + +**Settings → Display → Video Presentation** defaults to **Software**, which writes directly to the +framebuffer when available and performed better in xemu. **GPU** uses NV2A color conversion and scaling +with automatic software fallback. The setting is saved as `streaming.gpu_presentation` and both paths +are available in the same build. + ## Streaming controls and statistics +The default stream profile is **640×480 at 60 FPS and 3 Mbps**. Saved settings take precedence. Changing **Settings → Display → Stream Resolution** or **Stream Frame Rate** resets **Stream Bitrate** to a default calculated with Moonlight Android's resolution factors and pixel-count interpolation, scaled by the selected FPS and rounded to whole Mbps. At 30 / 60 FPS, the defaults are 1 / 3 Mbps for @@ -158,12 +166,9 @@ including any scanout wait. The stream menu can toggle these statistics for the current session; the saved setting sets the initial state of the next session. -Video decoding runs on the CPU. Optional experimental GPU presentation accelerates color conversion -and scaling, with software fallback for unsupported frames or presentation failures. GPU -presentation is enabled by default; the build options below control its use. - -Every stream summary reports GPU and software frame counts. **SOFTWARE FALLBACK USED** includes -the reason software presentation was selected, even when Performance Stats is disabled. +Video decoding runs on the CPU. Every stream summary reports GPU and software frame counts. +**SOFTWARE FALLBACK USED** includes the reason GPU presentation switched to software, even when +Performance Stats is disabled. ## 🛠️ Build @@ -260,14 +265,13 @@ cmake -S . -B cmake-build-release -DBUILD_DOCS=OFF -DCMAKE_BUILD_TYPE=Release Video build settings: -| CMake option | Default | Purpose | -| ----------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------- | -| `MOONLIGHT_ENABLE_GPU_PRESENTATION` | `ON` | Enable experimental NV2A color conversion and scaling with software fallback. | -| `MOONLIGHT_BUILD_VIDEO_EXPERIMENTS` | `OFF` | Build the standalone probes described in [Video experiments](experiments/README.md). | -| `MOONLIGHT_INPUT_DIAGNOSTICS` | `OFF` | Show controller state and event counters for troubleshooting. Requires GPU presentation to be disabled so the strip is visible. | +| CMake option | Default | Purpose | +| ----------------------------------- | ------- | ---------------------------------------------------------------------------------------------- | +| `MOONLIGHT_BUILD_VIDEO_EXPERIMENTS` | `OFF` | Build the standalone probes described in [Video experiments](experiments/README.md). | +| `MOONLIGHT_INPUT_DIAGNOSTICS` | `OFF` | Show controller state and event counters. Select Software presentation to see the debug strip. | -Pass options when configuring, for example `-DMOONLIGHT_ENABLE_GPU_PRESENTATION=OFF` for software-only -presentation. Existing build directories retain their cached settings until explicitly reconfigured. +Pass these options when configuring. Existing build directories retain cached build options until +explicitly reconfigured. ### Build diff --git a/cmake/nxdk.cmake b/cmake/nxdk.cmake index 47036e3..210e40b 100644 --- a/cmake/nxdk.cmake +++ b/cmake/nxdk.cmake @@ -380,7 +380,6 @@ function(moonlight_add_xbox_build) -DMOONLIGHT_TOOLCHAIN_FILE:FILEPATH=${toolchain_file} -DMOONLIGHT_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} -DMOONLIGHT_BUILD_VIDEO_EXPERIMENTS:BOOL=${MOONLIGHT_BUILD_VIDEO_EXPERIMENTS} - -DMOONLIGHT_ENABLE_GPU_PRESENTATION:BOOL=${MOONLIGHT_ENABLE_GPU_PRESENTATION} -DMOONLIGHT_INPUT_DIAGNOSTICS:BOOL=${MOONLIGHT_INPUT_DIAGNOSTICS} -P "${driver_script}" ) diff --git a/cmake/run-child-build.cmake b/cmake/run-child-build.cmake index a21ea26..73a1088 100644 --- a/cmake/run-child-build.cmake +++ b/cmake/run-child-build.cmake @@ -29,7 +29,6 @@ if(MOONLIGHT_COMMAND_MODE STREQUAL "configure") -B "${MOONLIGHT_BINARY_DIR}" -DMOONLIGHT_BUILD_KIND:STRING=XBOX "-DMOONLIGHT_BUILD_VIDEO_EXPERIMENTS:BOOL=${MOONLIGHT_BUILD_VIDEO_EXPERIMENTS}" - "-DMOONLIGHT_ENABLE_GPU_PRESENTATION:BOOL=${MOONLIGHT_ENABLE_GPU_PRESENTATION}" "-DMOONLIGHT_INPUT_DIAGNOSTICS:BOOL=${MOONLIGHT_INPUT_DIAGNOSTICS}" "-DMOONLIGHT_NXDK_DIR:PATH=${MOONLIGHT_NXDK_SOURCE_DIR}" "-DMOONLIGHT_NXDK_BUILD_DIR:PATH=${MOONLIGHT_NXDK_DIR}" diff --git a/cmake/sources.cmake b/cmake/sources.cmake index f1b74c8..00ded9a 100644 --- a/cmake/sources.cmake +++ b/cmake/sources.cmake @@ -18,6 +18,7 @@ set(MOONLIGHT_TEST_EXCLUDED_SOURCES "${MOONLIGHT_SOURCE_ROOT}/src/main.cpp" "${MOONLIGHT_SOURCE_ROOT}/src/splash/splash_screen.cpp" "${MOONLIGHT_SOURCE_ROOT}/src/streaming/ffmpeg_stream_backend.cpp" + "${MOONLIGHT_SOURCE_ROOT}/src/streaming/ffmpeg_video_frame.cpp" "${MOONLIGHT_SOURCE_ROOT}/src/startup/memory_stats.cpp" "${MOONLIGHT_SOURCE_ROOT}/src/streaming/session.cpp" "${MOONLIGHT_SOURCE_ROOT}/src/streaming/stream_controls.cpp" diff --git a/cmake/xbox-build.cmake b/cmake/xbox-build.cmake index bfb1429..64d204b 100644 --- a/cmake/xbox-build.cmake +++ b/cmake/xbox-build.cmake @@ -116,8 +116,6 @@ target_compile_definitions(${CMAKE_PROJECT_NAME} USBH_USE_EXTERNAL_CONFIG="usbh_config_xbox.h" TOML_EXCEPTIONS=0 TOML_ENABLE_WINDOWS_COMPAT=0) -target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE - MOONLIGHT_ENABLE_GPU_PRESENTATION=$) target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE MOONLIGHT_INPUT_DIAGNOSTICS=$) add_dependencies(${CMAKE_PROJECT_NAME} moonlight-common-c) diff --git a/experiments/README.md b/experiments/README.md index 1c44419..a13c814 100644 --- a/experiments/README.md +++ b/experiments/README.md @@ -27,8 +27,8 @@ Launch an ISO with the repository's configured xemu installation, for example: ``` From a Windows command prompt, use `scripts\run-xemu.cmd --iso` with the same ISO path. -The probes use GPU presentation directly; `MOONLIGHT_ENABLE_GPU_PRESENTATION` controls the -streaming client independently. +The probes use GPU presentation directly. The streaming client selects its path in +**Settings → Display → Video Presentation**. ## Presentation probe diff --git a/src/app/client_state.cpp b/src/app/client_state.cpp index a9b1c31..61f01d6 100644 --- a/src/app/client_state.cpp +++ b/src/app/client_state.cpp @@ -32,8 +32,6 @@ namespace { constexpr const char *SETTINGS_CATEGORY_PREFIX = "settings-category:"; constexpr std::array ADD_HOST_ADDRESS_KEYPAD_CHARACTERS {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0'}; constexpr std::array ADD_HOST_PORT_KEYPAD_CHARACTERS {'1', '2', '3', '4', '5', '6', '7', '8', '9', '0'}; - constexpr int DEFAULT_STREAM_FRAMERATE = 30; - constexpr int DEFAULT_STREAM_BITRATE_KBPS = 1000; constexpr std::array STREAM_FRAMERATE_OPTIONS {15, 20, 24, 25, 30, 60}; constexpr std::array STREAM_BITRATE_OPTIONS {500, 750, 1000, 1500, 2000, 2500, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, 11000, 12000, 13000, 14000, 15000, 16000, 17000, 18000, 19000, 20000}; @@ -230,7 +228,7 @@ namespace { void cycle_stream_framerate(app::ClientState &state) { const auto current = std::find(STREAM_FRAMERATE_OPTIONS.begin(), STREAM_FRAMERATE_OPTIONS.end(), state.settings.streamFramerate); if (current == STREAM_FRAMERATE_OPTIONS.end()) { - state.settings.streamFramerate = DEFAULT_STREAM_FRAMERATE; + state.settings.streamFramerate = app::DEFAULT_STREAM_FRAMERATE; } else { const std::size_t nextIndex = (static_cast(std::distance(STREAM_FRAMERATE_OPTIONS.begin(), current)) + 1U) % STREAM_FRAMERATE_OPTIONS.size(); state.settings.streamFramerate = STREAM_FRAMERATE_OPTIONS[nextIndex]; @@ -598,6 +596,12 @@ namespace { "Cycle through the preferred video bitrate. Lower bitrates reduce bandwidth use. Changing resolution or frame rate resets this value to a recommended default.", true, }, + { + "toggle-gpu-presentation", + std::string("Video Presentation: ") + (state.settings.presentation.gpuPresentation ? "GPU" : "Software"), + "Choose how new streams display decoded video. GPU uses NV2A with automatic software fallback. Software can be faster in xemu.", + true, + }, { "toggle-play-audio-on-pc", std::string("Play Audio on PC: ") + (state.settings.playAudioOnPc ? "On" : "Off"), @@ -612,7 +616,7 @@ namespace { }, { "toggle-show-performance-stats", - std::string("Performance Stats: ") + (state.settings.showPerformanceStats ? "On" : "Off"), + std::string("Performance Stats: ") + (state.settings.presentation.showPerformanceStats ? "On" : "Off"), "Show performance statistics during playback and after the stream ends.", true, }, @@ -1892,10 +1896,7 @@ namespace app { state.settings.logViewerPlacement = LogViewerPlacement::full; state.settings.loggingLevel = logging::LogLevel::none; state.settings.xemuConsoleLoggingLevel = logging::LogLevel::none; - state.settings.streamFramerate = DEFAULT_STREAM_FRAMERATE; - state.settings.streamBitrateKbps = DEFAULT_STREAM_BITRATE_KBPS; state.settings.playAudioOnPc = false; - state.settings.showPerformanceStats = false; state.settings.playAudioOnXbox = true; state.settings.controls.screenshotDirectory.clear(); state.settings.controls.keyboardDockTop = false; @@ -2447,6 +2448,32 @@ namespace app { return true; } + /** + * @brief Toggle a presentation preference and refresh its selected menu row. + * + * @param state Client state containing the presentation preferences. + * @param itemId Activated menu row. + * @param update Update receiving the request to persist the change. + * @return True when the activated row changed a presentation preference. + */ + bool handle_presentation_setting_activation(ClientState &state, const std::string &itemId, AppUpdate &update) { + auto &presentation = state.settings.presentation; + if (itemId == "toggle-gpu-presentation") { + presentation.gpuPresentation = !presentation.gpuPresentation; + state.shell.statusMessage = std::string("Video presentation set to ") + (presentation.gpuPresentation ? "GPU" : "Software"); + } else if (itemId == "toggle-show-performance-stats") { + presentation.showPerformanceStats = !presentation.showPerformanceStats; + state.shell.statusMessage = std::string("Performance stats ") + (presentation.showPerformanceStats ? "enabled" : "disabled"); + } else { + return false; + } + + state.settings.dirty = true; + update.persistence.settingsChanged = true; + rebuild_menu(state, itemId); + return true; + } + /** * @brief Update settings inline or open the screenshot folder picker, emitting only required side effects. * @@ -2504,6 +2531,9 @@ namespace app { rebuild_menu(state, "cycle-stream-bitrate"); return; } + if (handle_presentation_setting_activation(state, detailUpdate.activatedItemId, *update)) { + return; + } if (detailUpdate.activatedItemId == "toggle-play-audio-on-pc") { state.settings.playAudioOnPc = !state.settings.playAudioOnPc; state.settings.dirty = true; @@ -2520,14 +2550,6 @@ namespace app { rebuild_menu(state, "toggle-play-audio-on-xbox"); return; } - if (detailUpdate.activatedItemId == "toggle-show-performance-stats") { - state.settings.showPerformanceStats = !state.settings.showPerformanceStats; - state.settings.dirty = true; - update->persistence.settingsChanged = true; - state.shell.statusMessage = std::string("Performance stats ") + (state.settings.showPerformanceStats ? "enabled" : "disabled"); - rebuild_menu(state, "toggle-show-performance-stats"); - return; - } if (detailUpdate.activatedItemId == "screenshot-directory") { state.directoryPicker = {}; state.directoryPicker.path = state.settings.controls.screenshotDirectory.empty() ? startup::default_storage_path("screenshots") : state.settings.controls.screenshotDirectory; diff --git a/src/app/client_state.h b/src/app/client_state.h index 9326a35..ab91668 100644 --- a/src/app/client_state.h +++ b/src/app/client_state.h @@ -75,6 +75,9 @@ namespace app { right, ///< Dock the log viewer on the right side of the split layout. }; + inline constexpr int DEFAULT_STREAM_FRAMERATE = 60; ///< Stream frame rate at the default 640x480 resolution. + inline constexpr int DEFAULT_STREAM_BITRATE_KBPS = 3000; ///< Bitrate for the default 640x480, 60 FPS profile. + /** * @brief Focus areas used by the two-pane settings screen. */ @@ -378,6 +381,14 @@ namespace app { bool keyboardDockTop = false; ///< Dock the stream keyboard to the top instead of the bottom. }; + /** + * @brief Preferences for presenting decoded video and its on-screen statistics. + */ + struct VideoPresentationPreferences { + bool gpuPresentation = false; ///< Opt in to NV2A presentation with software fallback on new streams. + bool showPerformanceStats = false; ///< Display live performance telemetry and include it in the stream summary. + }; + /** * @brief State owned by the settings, log viewer, and saved-file workflows. */ @@ -393,10 +404,10 @@ namespace app { std::vector availableVideoModes; ///< Detected Xbox video modes exposed as stream-resolution choices. VIDEO_MODE preferredVideoMode {}; ///< Preferred stream resolution requested from the host. bool preferredVideoModeSet = false; ///< True when preferredVideoMode contains a user-selected or default mode. - int streamFramerate = 30; ///< Preferred stream frame rate in frames per second. - int streamBitrateKbps = 1000; ///< Preferred stream bitrate in kilobits per second. + int streamFramerate = DEFAULT_STREAM_FRAMERATE; ///< Preferred stream frame rate in frames per second. + int streamBitrateKbps = DEFAULT_STREAM_BITRATE_KBPS; ///< Preferred stream bitrate in kilobits per second. + VideoPresentationPreferences presentation; ///< Presentation path and statistics overlay preferences. bool playAudioOnPc = false; ///< True when the host PC should continue local audio playback during streaming. - bool showPerformanceStats = false; ///< Display live performance telemetry and include it in the stream summary. bool playAudioOnXbox = true; ///< True when the Xbox should decode and play streamed audio locally. LocalStreamPreferences controls; ///< Screenshot storage and keyboard placement preferences. bool dirty = false; ///< True when persisted TOML-backed settings changed and should be saved. @@ -508,7 +519,7 @@ namespace app { }; /** - * @brief Create the initial app state shown after startup. + * @brief Create the initial app state with the default stream preferences. * * @return The initial client state. */ diff --git a/src/app/settings_storage.cpp b/src/app/settings_storage.cpp index c97b25e..a0c8b71 100644 --- a/src/app/settings_storage.cpp +++ b/src/app/settings_storage.cpp @@ -31,6 +31,7 @@ extern "C" FILE *_wfopen(const wchar_t *path, const wchar_t *mode); #include // local includes +#include "src/app/stream_bitrate.h" #include "src/platform/error_utils.h" #include "src/platform/filesystem_utils.h" #include "src/startup/storage_paths.h" @@ -348,6 +349,7 @@ namespace { content += "# Preferred streaming parameters.\n"; content += std::string("fps = ") + std::to_string(settings.streamFramerate) + "\n"; content += std::string("bitrate_kbps = ") + std::to_string(settings.streamBitrateKbps) + "\n"; + content += std::string("gpu_presentation = ") + (settings.gpuPresentation ? "true" : "false") + "\n"; content += std::string("play_audio_on_pc = ") + (settings.playAudioOnPc ? "true" : "false") + "\n"; content += std::string("play_audio_on_xbox = ") + (settings.playAudioOnXbox ? "true" : "false") + "\n"; content += "# Show stream telemetry after streaming ends.\n"; @@ -399,7 +401,7 @@ namespace { void inspect_streaming_keys(const toml::table &streamingTable, const std::string &filePath, app::LoadAppSettingsResult *result) { for (const auto &[rawKey, node] : streamingTable) { const std::string key(rawKey.str()); - if (key == "video_width" || key == "video_height" || key == "video_bpp" || key == "video_refresh" || key == "video_mode_selected" || key == "fps" || key == "bitrate_kbps" || key == "play_audio_on_pc" || key == "play_audio_on_xbox" || key == "show_performance_stats" || key == "screenshot_directory" || key == "keyboard_dock_top") { + if (key == "video_width" || key == "video_height" || key == "video_bpp" || key == "video_refresh" || key == "video_mode_selected" || key == "fps" || key == "bitrate_kbps" || key == "gpu_presentation" || key == "play_audio_on_pc" || key == "play_audio_on_xbox" || key == "show_performance_stats" || key == "screenshot_directory" || key == "keyboard_dock_top") { continue; } @@ -496,8 +498,12 @@ namespace app { load_integer_setting(settingsTable["streaming"]["video_refresh"], filePath, "streaming.video_refresh", &result.settings.preferredVideoMode.refresh, &result.warnings); load_boolean_setting(settingsTable["streaming"]["video_mode_selected"], filePath, "streaming.video_mode_selected", &result.settings.preferredVideoModeSet, &result.warnings); load_integer_setting(settingsTable["streaming"]["fps"], filePath, "streaming.fps", &result.settings.streamFramerate, &result.warnings); + const auto &preferredMode = result.settings.preferredVideoMode; + const VIDEO_MODE bitrateMode = result.settings.preferredVideoModeSet && preferredMode.width > 0 && preferredMode.height > 0 ? preferredMode : VIDEO_MODE {640, 480, 32, 60}; + result.settings.streamBitrateKbps = default_stream_bitrate_kbps(bitrateMode.width, bitrateMode.height, result.settings.streamFramerate); load_integer_setting(settingsTable["streaming"]["bitrate_kbps"], filePath, "streaming.bitrate_kbps", &result.settings.streamBitrateKbps, &result.warnings); load_boolean_setting(settingsTable["streaming"]["play_audio_on_pc"], filePath, "streaming.play_audio_on_pc", &result.settings.playAudioOnPc, &result.warnings); + load_boolean_setting(settingsTable["streaming"]["gpu_presentation"], filePath, "streaming.gpu_presentation", &result.settings.gpuPresentation, &result.warnings); load_boolean_setting(settingsTable["streaming"]["play_audio_on_xbox"], filePath, "streaming.play_audio_on_xbox", &result.settings.playAudioOnXbox, &result.warnings); load_boolean_setting(settingsTable["streaming"]["show_performance_stats"], filePath, "streaming.show_performance_stats", &result.settings.showPerformanceStats, &result.warnings); load_boolean_setting(settingsTable["streaming"]["keyboard_dock_top"], filePath, "streaming.keyboard_dock_top", &result.settings.keyboardDockTop, &result.warnings); diff --git a/src/app/settings_storage.h b/src/app/settings_storage.h index 5f926ca..152bbb8 100644 --- a/src/app/settings_storage.h +++ b/src/app/settings_storage.h @@ -22,13 +22,14 @@ namespace app { app::LogViewerPlacement logViewerPlacement = app::LogViewerPlacement::full; ///< Preferred placement for the in-app log viewer. VIDEO_MODE preferredVideoMode {}; ///< Preferred stream resolution requested from the host. bool preferredVideoModeSet = false; ///< True when preferredVideoMode contains a saved user preference. - int streamFramerate = 30; ///< Preferred stream frame rate in frames per second. - int streamBitrateKbps = 1000; ///< Preferred stream bitrate in kilobits per second. + int streamFramerate = DEFAULT_STREAM_FRAMERATE; ///< Preferred stream frame rate in frames per second. + int streamBitrateKbps = DEFAULT_STREAM_BITRATE_KBPS; ///< Preferred stream bitrate in kilobits per second. bool playAudioOnPc = false; ///< True when the host PC should continue local audio playback during streaming. bool showPerformanceStats = false; ///< Display live performance telemetry and include it in the stream summary. bool playAudioOnXbox = true; ///< True when the Xbox should decode and play streamed audio locally. std::string screenshotDirectory; ///< Screenshot folder; empty selects the title-scoped screenshots directory. bool keyboardDockTop = false; ///< Dock the stream keyboard to the top instead of the bottom. + bool gpuPresentation = false; ///< Opt in to NV2A presentation with software fallback. }; /** @@ -57,7 +58,7 @@ namespace app { std::string default_settings_path(); /** - * @brief Load the persisted Moonlight settings file. + * @brief Load saved preferences and derive a bitrate from the stream profile when no bitrate is saved. * * @param filePath Settings file to read. * @return Loaded settings plus any non-fatal warnings. diff --git a/src/main.cpp b/src/main.cpp index 5a1c9af..47f0a02 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -46,8 +46,9 @@ namespace { state.settings.preferredVideoModeSet = settings.preferredVideoModeSet; state.settings.streamFramerate = settings.streamFramerate; state.settings.streamBitrateKbps = settings.streamBitrateKbps; + state.settings.presentation.gpuPresentation = settings.gpuPresentation; state.settings.playAudioOnPc = settings.playAudioOnPc; - state.settings.showPerformanceStats = settings.showPerformanceStats; + state.settings.presentation.showPerformanceStats = settings.showPerformanceStats; state.settings.playAudioOnXbox = settings.playAudioOnXbox; state.settings.controls.screenshotDirectory = settings.screenshotDirectory; state.settings.controls.keyboardDockTop = settings.keyboardDockTop; diff --git a/src/streaming/ffmpeg_stream_backend.cpp b/src/streaming/ffmpeg_stream_backend.cpp index f4851ca..863323f 100644 --- a/src/streaming/ffmpeg_stream_backend.cpp +++ b/src/streaming/ffmpeg_stream_backend.cpp @@ -24,7 +24,6 @@ extern "C" { #include #include #include -#include #include #include #include @@ -452,8 +451,9 @@ namespace streaming { } void FfmpegStreamBackend::local_overlay_size(int &width, int &height) const { - if (video_.renderFrame.width > 0 && video_.renderFrame.height > 0) { - const auto destination = build_letterboxed_destination(width, height, video_.renderFrame.width, video_.renderFrame.height); + const auto &frame = video_.renderFrame.view(); + if (frame.width > 0 && frame.height > 0) { + const auto destination = build_letterboxed_destination(width, height, frame.width, frame.height); width = destination.w; height = destination.h; } @@ -463,26 +463,13 @@ namespace streaming { if (const auto result = take_screenshot_result(); !result.empty()) { logging::info("screenshots", result); } - if (screenshot_.thread != nullptr || video_.renderFrame.width <= 0 || video_.renderFrame.height <= 0) { + if (screenshot_.thread != nullptr || video_.renderFrame.view().width <= 0) { logging::warn("screenshots", "Capture not queued: another screenshot is pending or no video frame is available"); return false; } - // The event thread owns renderFrame. Copy only its planar YUV payload here; - // conversion, directory creation, BMP writes and indexing belong to the worker. - const auto &frame = video_.renderFrame; - auto &snapshot = screenshot_.frame; - snapshot.width = frame.width; - snapshot.height = frame.height; - snapshot.yPitch = frame.yPitch; - snapshot.uPitch = frame.uPitch; - snapshot.vPitch = frame.vPitch; - snapshot.matrix = frame.matrix; - snapshot.fullRange = frame.fullRange; - const std::array sources {&frame.yPlane, &frame.uPlane, &frame.vPlane}; - const std::array destinations {&snapshot.yPlane, &snapshot.uPlane, &snapshot.vPlane}; - for (std::size_t i = 0; i < sources.size(); ++i) { - destinations[i]->resize(sources[i]->size()); - std::memcpy(destinations[i]->data(), sources[i]->data(), sources[i]->size()); + if (!screenshot_.frame.retain(video_.renderFrame)) { + logging::warn("screenshots", "Capture not queued: could not retain the decoded frame"); + return false; } screenshot_.source = source; screenshot_.directory = directory; @@ -496,7 +483,7 @@ namespace streaming { "save-screenshot", &screenshot_); if (screenshot_.thread == nullptr) { - screenshot_.frame = {}; + screenshot_.frame.clear(); logging::warn("screenshots", std::string("Cannot start screenshot worker: ") + SDL_GetError()); } return screenshot_.thread != nullptr; @@ -541,7 +528,7 @@ namespace streaming { return saved; } - SDL_Surface *FfmpegStreamBackend::convert_screenshot_frame(const LatestVideoFrame &frame) { + SDL_Surface *FfmpegStreamBackend::convert_screenshot_frame(const Yuv420FrameView &frame) { SDL_Surface *surface = SDL_CreateRGBSurfaceWithFormat(0, frame.width, frame.height, 24, SDL_PIXELFORMAT_BGR24); if (surface == nullptr) { return nullptr; @@ -553,8 +540,8 @@ namespace streaming { } const int *coefficients = sws_getCoefficients(frame.matrix == VideoColorMatrix::bt709 ? SWS_CS_ITU709 : SWS_CS_ITU601); sws_setColorspaceDetails(conversion, coefficients, frame.fullRange, coefficients, 1, 0, 1 << 16, 1 << 16); - const std::array pixels {frame.yPlane.data(), frame.uPlane.data(), frame.vPlane.data(), nullptr}; - const std::array pitch {frame.yPitch, frame.uPitch, frame.vPitch, 0}; + const std::array pixels {frame.planes[0], frame.planes[1], frame.planes[2], nullptr}; + const std::array pitch {frame.pitches[0], frame.pitches[1], frame.pitches[2], 0}; std::array output {static_cast(surface->pixels), nullptr, nullptr, nullptr}; const std::array outputPitch {surface->pitch, 0, 0, 0}; const bool converted = sws_scale(conversion, pixels.data(), pitch.data(), 0, frame.height, output.data(), outputPitch.data()) == frame.height; @@ -573,9 +560,9 @@ namespace streaming { job.progress.store(1); std::string error; const std::string path = app::next_screenshot_path(job.directory, job.capturedAt, error); - SDL_Surface *surface = path.empty() ? nullptr : convert_screenshot_frame(job.frame); + SDL_Surface *surface = path.empty() ? nullptr : convert_screenshot_frame(job.frame.view()); // Release the YUV snapshot before disk I/O so full-screen menus still fit in 64 MiB. - job.frame = {}; + job.frame.clear(); bool saved = false; if (surface != nullptr) { job.progress.store(10); @@ -627,6 +614,10 @@ namespace streaming { audioPlaybackEnabled_.store(enabled); } + void FfmpegStreamBackend::set_gpu_presentation_enabled(bool enabled) { + gpuPresentationEnabled_ = enabled; + } + void FfmpegStreamBackend::shutdown() { cleanup_audio_decoder(); cleanup_video_decoder(); @@ -634,7 +625,7 @@ namespace streaming { SDL_WaitThread(screenshot_.thread, nullptr); screenshot_.thread = nullptr; logging::info("screenshots", screenshot_.result); - screenshot_.frame = {}; + screenshot_.frame.clear(); } } @@ -739,36 +730,26 @@ namespace streaming { bool textureNeedsUpload = false; if (const std::uint64_t publishedFrameVersion = video_.publishedFrameVersion.load(); publishedFrameVersion != video_.renderedFrameVersion) { std::scoped_lock lock(video_.frame_mutex()); - if (video_.latestFrameVersion != video_.renderedFrameVersion && video_.latestFrame.width > 0 && video_.latestFrame.height > 0) { + if (video_.latestFrameVersion != video_.renderedFrameVersion && video_.latestFrame.view().width > 0) { if (video_.latestFrameVersion > video_.renderedFrameVersion + 1) { statistics_.dropped(video_.latestFrameVersion - video_.renderedFrameVersion - 1); } - std::swap(video_.renderFrame, video_.latestFrame); + video_.renderFrame.swap(video_.latestFrame); video_.renderedFrameVersion = video_.latestFrameVersion; textureNeedsUpload = true; } } - if (video_.renderFrame.width <= 0 || video_.renderFrame.height <= 0) { + const auto &frame = video_.renderFrame.view(); + if (frame.width <= 0 || frame.height <= 0) { return false; } #ifdef NXDK - if (allowDirectFramebuffer && !overlayFailed_) { + if (allowDirectFramebuffer && gpuPresentationEnabled_ && !overlayFailed_) { if (!textureNeedsUpload && !statisticsBitmapChanged_ && presenter_.active()) { return true; } - const LatestVideoFrame &frame = video_.renderFrame; - const Yuv420FrameView view { - frame.width, - frame.height, - {frame.yPlane.data(), frame.uPlane.data(), frame.vPlane.data()}, - {frame.yPlane.size(), frame.uPlane.size(), frame.vPlane.size()}, - {frame.yPitch, frame.uPitch, frame.vPitch}, - frame.matrix, - frame.fullRange, - frame.interlaced, - }; const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, frame.width, frame.height); if (!presenter_.active()) { const VIDEO_MODE mode = XVideoGetMode(); @@ -778,7 +759,7 @@ namespace streaming { } } std::string reason; - if (presenter_.present(view, {destination.x, destination.y, destination.w, destination.h}, &reason, {}, &statisticsOverlay_)) { + if (presenter_.present(frame, {destination.x, destination.y, destination.w, destination.h}, &reason, {}, &statisticsOverlay_)) { if (textureNeedsUpload) { ++acceleration_.gpuFrames; } @@ -798,29 +779,29 @@ namespace streaming { (void) allowDirectFramebuffer; #endif - if (video_.texture == nullptr || video_.textureWidth != video_.renderFrame.width || video_.textureHeight != video_.renderFrame.height) { + if (video_.texture == nullptr || video_.textureWidth != frame.width || video_.textureHeight != frame.height) { if (video_.texture != nullptr) { SDL_DestroyTexture(video_.texture); video_.texture = nullptr; } - video_.texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING, video_.renderFrame.width, video_.renderFrame.height); + video_.texture = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING, frame.width, frame.height); if (video_.texture == nullptr) { logging::error("stream", std::string("SDL_CreateTexture failed for video presentation: ") + SDL_GetError()); return false; } - video_.textureWidth = video_.renderFrame.width; - video_.textureHeight = video_.renderFrame.height; + video_.textureWidth = frame.width; + video_.textureHeight = frame.height; textureNeedsUpload = true; } - if (textureNeedsUpload && SDL_UpdateYUVTexture(video_.texture, nullptr, reinterpret_cast(video_.renderFrame.yPlane.data()), video_.renderFrame.yPitch, reinterpret_cast(video_.renderFrame.uPlane.data()), video_.renderFrame.uPitch, reinterpret_cast(video_.renderFrame.vPlane.data()), video_.renderFrame.vPitch) != 0) { + if (textureNeedsUpload && SDL_UpdateYUVTexture(video_.texture, nullptr, frame.planes[0], frame.pitches[0], frame.planes[1], frame.pitches[1], frame.planes[2], frame.pitches[2]) != 0) { logging::error("stream", std::string("SDL_UpdateYUVTexture failed during video presentation: ") + SDL_GetError()); return false; } - const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, video_.renderFrame.width, video_.renderFrame.height); + const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, frame.width, frame.height); const bool presented = SDL_RenderCopy(renderer, video_.texture, nullptr, &destination) == 0; if (presented) { render_sdl_statistics(renderer, destination); @@ -870,12 +851,9 @@ namespace streaming { cleanup_video_decoder(); acceleration_ = {}; + acceleration_.gpuPresentationRequested = gpuPresentationEnabled_; statistics_.reset(); overlayFailed_ = false; -#if defined(NXDK) && !MOONLIGHT_ENABLE_GPU_PRESENTATION - overlayFailed_ = true; - acceleration_.record_fallback("GPU presentation disabled in this build"); -#endif const int cpuFlags = av_get_cpu_flags(); acceleration_.cpuFeatures = (cpuFlags & AV_CPU_FLAG_SSE) != 0 ? "MMX/SSE" : ((cpuFlags & AV_CPU_FLAG_MMX) != 0 ? "MMX" : "Scalar"); acceleration_.threaded = std::string(avcodec_configuration()).find("--enable-w32threads") != std::string::npos; @@ -992,13 +970,12 @@ namespace streaming { video_.presentScaleContext = nullptr; } - video_.convertedBuffer.clear(); video_.packetBuffer.clear(); - video_.renderFrame = LatestVideoFrame {}; - video_.decodeFrame = LatestVideoFrame {}; + video_.renderFrame.clear(); + video_.decodeFrame.clear(); { std::scoped_lock lock(video_.frame_mutex()); - video_.latestFrame = LatestVideoFrame {}; + video_.latestFrame.clear(); video_.latestFrameVersion = 0; } video_.renderedFrameVersion = 0; @@ -1118,51 +1095,17 @@ namespace streaming { return DR_NEED_IDR; } - bool FfmpegStreamBackend::publish_video_frame(const AVFrame *frameToPresent) { - if (frameToPresent == nullptr || frameToPresent->width <= 0 || frameToPresent->height <= 0 || frameToPresent->linesize[0] <= 0 || frameToPresent->linesize[1] <= 0 || frameToPresent->linesize[2] <= 0 || frameToPresent->data[0] == nullptr || frameToPresent->data[1] == nullptr || frameToPresent->data[2] == nullptr) { + bool FfmpegStreamBackend::publish_video_frame(AVFrame &frameToPresent) { + if (!video_.decodeFrame.take(frameToPresent)) { return false; } - - LatestVideoFrame &nextFrame = video_.decodeFrame; - nextFrame.width = frameToPresent->width; - nextFrame.height = frameToPresent->height; - nextFrame.yPitch = frameToPresent->linesize[0]; - nextFrame.uPitch = frameToPresent->linesize[1]; - nextFrame.vPitch = frameToPresent->linesize[2]; - nextFrame.fullRange = frameToPresent->color_range == AVCOL_RANGE_JPEG; - nextFrame.interlaced = (frameToPresent->flags & AV_FRAME_FLAG_INTERLACED) != 0; - switch (frameToPresent->colorspace) { - case AVCOL_SPC_BT709: - nextFrame.matrix = VideoColorMatrix::bt709; - break; - case AVCOL_SPC_UNSPECIFIED: - case AVCOL_SPC_BT470BG: - case AVCOL_SPC_SMPTE170M: - nextFrame.matrix = VideoColorMatrix::bt601; - break; - default: - nextFrame.matrix = VideoColorMatrix::unsupported; - break; - } - - const int chromaHeight = (frameToPresent->height + 1) / 2; - const std::size_t yPlaneSize = static_cast(frameToPresent->linesize[0]) * static_cast(frameToPresent->height); - const std::size_t uPlaneSize = static_cast(frameToPresent->linesize[1]) * static_cast(chromaHeight); - const std::size_t vPlaneSize = static_cast(frameToPresent->linesize[2]) * static_cast(chromaHeight); - - nextFrame.yPlane.resize(yPlaneSize); - nextFrame.uPlane.resize(uPlaneSize); - nextFrame.vPlane.resize(vPlaneSize); - std::memcpy(nextFrame.yPlane.data(), frameToPresent->data[0], nextFrame.yPlane.size()); - std::memcpy(nextFrame.uPlane.data(), frameToPresent->data[1], nextFrame.uPlane.size()); - std::memcpy(nextFrame.vPlane.data(), frameToPresent->data[2], nextFrame.vPlane.size()); - { std::scoped_lock lock(video_.frame_mutex()); - std::swap(video_.latestFrame, video_.decodeFrame); + video_.latestFrame.swap(video_.decodeFrame); ++video_.latestFrameVersion; video_.publishedFrameVersion.store(video_.latestFrameVersion); } + video_.decodeFrame.clear(); video_.lastDecodedFrameUs.store(PltGetMicroseconds()); video_.hasFrame.store(true); @@ -1183,7 +1126,8 @@ namespace streaming { const int framebufferWidth = screenWidth > 0 ? std::min(screenWidth, videoMode.width) : videoMode.width; const int framebufferHeight = screenHeight > 0 ? std::min(screenHeight, videoMode.height) : videoMode.height; - const SDL_Rect destination = build_letterboxed_destination(framebufferWidth, framebufferHeight, video_.renderFrame.width, video_.renderFrame.height); + const auto &frame = video_.renderFrame.view(); + const SDL_Rect destination = build_letterboxed_destination(framebufferWidth, framebufferHeight, frame.width, frame.height); if (destination.w <= 0 || destination.h <= 0) { return false; } @@ -1197,8 +1141,8 @@ namespace streaming { video_.presentScaleContext = sws_getCachedContext( video_.presentScaleContext, - video_.renderFrame.width, - video_.renderFrame.height, + frame.width, + frame.height, AV_PIX_FMT_YUV420P, destination.w, destination.h, @@ -1214,15 +1158,15 @@ namespace streaming { } const std::array sourceData { - video_.renderFrame.yPlane.data(), - video_.renderFrame.uPlane.data(), - video_.renderFrame.vPlane.data(), + frame.planes[0], + frame.planes[1], + frame.planes[2], nullptr, }; const std::array sourceLinesize { - video_.renderFrame.yPitch, - video_.renderFrame.uPitch, - video_.renderFrame.vPitch, + frame.pitches[0], + frame.pitches[1], + frame.pitches[2], 0, }; std::array destinationData { @@ -1238,7 +1182,7 @@ namespace streaming { 0, }; - if (const int scaledRows = sws_scale(video_.presentScaleContext, sourceData.data(), sourceLinesize.data(), 0, video_.renderFrame.height, destinationData.data(), destinationLinesize.data()); scaledRows <= 0) { + if (const int scaledRows = sws_scale(video_.presentScaleContext, sourceData.data(), sourceLinesize.data(), 0, frame.height, destinationData.data(), destinationLinesize.data()); scaledRows <= 0) { logging::warn("stream", "sws_scale failed for direct framebuffer presentation"); return false; } @@ -1264,7 +1208,7 @@ namespace streaming { return receiveResult; } - const AVFrame *frameToPresent = video_.decodedFrame; + AVFrame *frameToPresent = video_.decodedFrame; if (video_.decodedFrame->format != AV_PIX_FMT_YUV420P) { video_.scaleContext = sws_getCachedContext( video_.scaleContext, @@ -1285,14 +1229,6 @@ namespace streaming { return AVERROR(EINVAL); } - const int bufferSize = av_image_get_buffer_size(AV_PIX_FMT_YUV420P, video_.decodedFrame->width, video_.decodedFrame->height, 32); - if (bufferSize <= 0) { - logging::warn("stream", "av_image_get_buffer_size failed for converted video frame"); - av_frame_unref(video_.decodedFrame); - return AVERROR(EINVAL); - } - - video_.convertedBuffer.resize(static_cast(bufferSize)); av_frame_unref(video_.convertedFrame); video_.convertedFrame->format = AV_PIX_FMT_YUV420P; video_.convertedFrame->width = video_.decodedFrame->width; @@ -1300,30 +1236,29 @@ namespace streaming { video_.convertedFrame->color_range = video_.decodedFrame->color_range; video_.convertedFrame->colorspace = video_.decodedFrame->colorspace; video_.convertedFrame->flags = video_.decodedFrame->flags; - if (const int fillResult = av_image_fill_arrays(video_.convertedFrame->data, video_.convertedFrame->linesize, video_.convertedBuffer.data(), AV_PIX_FMT_YUV420P, video_.decodedFrame->width, video_.decodedFrame->height, 32); fillResult < 0) { - logging::warn("stream", std::string("av_image_fill_arrays failed for converted frame: ") + describe_ffmpeg_error(fillResult)); + if (const int allocation = av_frame_get_buffer(video_.convertedFrame, 32); allocation < 0) { + logging::warn("stream", "av_frame_get_buffer failed for converted video frame"); av_frame_unref(video_.decodedFrame); - return fillResult; + return allocation; } - sws_scale( - video_.scaleContext, - video_.decodedFrame->data, - video_.decodedFrame->linesize, - 0, - video_.decodedFrame->height, - video_.convertedFrame->data, - video_.convertedFrame->linesize - ); + if (const int scaled = sws_scale(video_.scaleContext, video_.decodedFrame->data, video_.decodedFrame->linesize, 0, video_.decodedFrame->height, video_.convertedFrame->data, video_.convertedFrame->linesize); scaled != video_.convertedFrame->height) { + logging::warn("stream", "sws_scale failed to convert the full video frame"); + av_frame_unref(video_.decodedFrame); + av_frame_unref(video_.convertedFrame); + return scaled < 0 ? scaled : AVERROR(EINVAL); + } frameToPresent = video_.convertedFrame; } - if (!publish_video_frame(frameToPresent)) { + const bool published = publish_video_frame(*frameToPresent); + if (frameToPresent != video_.decodedFrame) { av_frame_unref(video_.decodedFrame); + } + if (!published) { + av_frame_unref(frameToPresent); return AVERROR(EINVAL); } - - av_frame_unref(video_.decodedFrame); } } diff --git a/src/streaming/ffmpeg_stream_backend.h b/src/streaming/ffmpeg_stream_backend.h index 260431d..3e5af60 100644 --- a/src/streaming/ffmpeg_stream_backend.h +++ b/src/streaming/ffmpeg_stream_backend.h @@ -20,6 +20,7 @@ // local includes #include "src/app/screenshot_library.h" #include "src/streaming/aligned_media_buffer.h" +#include "src/streaming/ffmpeg_video_frame.h" #include "src/streaming/pvideo_presenter.h" #include "src/streaming/stats_overlay.h" @@ -73,6 +74,12 @@ namespace streaming { */ void set_audio_playback_enabled(bool enabled); + /** + * @brief Select the preferred presentation path before starting the session. + * @param enabled True to try NV2A with automatic software fallback. + */ + void set_gpu_presentation_enabled(bool enabled); + /** * @brief Drain any pending screenshot and release FFmpeg, SDL, and cached frame resources. */ @@ -365,12 +372,12 @@ namespace streaming { int receive_available_video_frames(); /** - * @brief Publish one decoded or converted frame for SDL presentation. + * @brief Publish one decoded or converted frame for presentation. * - * @param frameToPresent FFmpeg frame in IYUV-compatible layout. - * @return True when the frame was copied into the presentation buffer. + * @param frameToPresent FFmpeg frame in I420 layout, emptied on success. + * @return True when the frame reference was transferred to the presenter queue. */ - bool publish_video_frame(const AVFrame *frameToPresent); + bool publish_video_frame(AVFrame &frameToPresent); /** * @brief Present the latest decoded frame through a platform-specific direct framebuffer path. @@ -381,23 +388,6 @@ namespace streaming { */ bool render_latest_video_frame_to_framebuffer(int screenWidth, int screenHeight); - /** - * @brief Hold the latest IYUV video frame ready for SDL upload. - */ - struct LatestVideoFrame { - int width = 0; - int height = 0; - int yPitch = 0; - int uPitch = 0; - int vPitch = 0; - AlignedMediaBuffer yPlane; ///< Aligned, padded luma data. - AlignedMediaBuffer uPlane; ///< Aligned, padded blue chroma data. - AlignedMediaBuffer vPlane; ///< Aligned, padded red chroma data. - VideoColorMatrix matrix = VideoColorMatrix::bt601; ///< Color matrix preserved across frame publication. - bool fullRange = false; ///< Source uses full-range YUV. - bool interlaced = false; ///< Source contains interlaced fields. - }; - /** * @brief Own one bounded screenshot job independently of live rendering and decoding. */ @@ -405,7 +395,7 @@ namespace streaming { SDL_Thread *thread = nullptr; ///< Owned worker, joined after completion or at shutdown. std::atomic complete = false; ///< Publishes the completed result before the UI can read it. std::atomic progress = 0; ///< Conversion, BMP write and index progress; completion is published separately. - LatestVideoFrame frame; ///< Independently owned snapshot of the uncomposited YUV frame. + FfmpegVideoFrame frame; ///< Separate reference to the uncomposited YUV frame. app::ScreenshotSource source; ///< Original host and app identity. std::string directory; ///< Destination at request time. std::int64_t capturedAt = 0; ///< UTC capture time. @@ -423,7 +413,7 @@ namespace streaming { * @param frame Stable screenshot snapshot. * @return Owned BGR surface, or nullptr when conversion fails. */ - static SDL_Surface *convert_screenshot_frame(const LatestVideoFrame &frame); + static SDL_Surface *convert_screenshot_frame(const Yuv420FrameView &frame); /** * @brief Hold FFmpeg and SDL state used by the video path. @@ -442,7 +432,6 @@ namespace streaming { int textureHeight = 0; std::uint64_t latestFrameVersion = 0; std::uint64_t renderedFrameVersion = 0; - AlignedMediaBuffer convertedBuffer; ///< Aligned planar conversion storage. std::vector packetBuffer; /** @@ -459,9 +448,9 @@ namespace streaming { public: SDL_Rect directFramebufferDestination {0, 0, 0, 0}; - LatestVideoFrame latestFrame; - LatestVideoFrame decodeFrame; - LatestVideoFrame renderFrame; + FfmpegVideoFrame latestFrame; ///< Mailbox slot exchanged under frameMutex_. + FfmpegVideoFrame decodeFrame; ///< Decoder-owned slot reused to publish each new reference. + FfmpegVideoFrame renderFrame; ///< Render-thread reference retained for presentation and UI redraws. bool directFramebufferCleared = false; std::atomic decoderStopRequested = false; std::atomic hasFrame = false; @@ -504,6 +493,7 @@ namespace streaming { bool statisticsTextureChanged_ = false; ///< SDL texture requires the current bitmap. VideoRectangle statisticsTextureDamage_ {}; ///< Pending SDL texture upload, accumulated until software presentation. VideoAccelerationReport acceleration_; ///< Session diagnostics survive decoder cleanup. + bool gpuPresentationEnabled_ = false; ///< Session opt-in applied before decoder setup. bool overlayFailed_ = false; ///< Avoid repeating a known failing hardware path this session. std::atomic audioPlaybackEnabled_ = false; }; diff --git a/src/streaming/ffmpeg_video_frame.cpp b/src/streaming/ffmpeg_video_frame.cpp new file mode 100644 index 0000000..86ab9b0 --- /dev/null +++ b/src/streaming/ffmpeg_video_frame.cpp @@ -0,0 +1,131 @@ +/** + * @file src/streaming/ffmpeg_video_frame.cpp + * @brief Transfers FFmpeg frame references without copying pixels. + */ +// class header include +#include "src/streaming/ffmpeg_video_frame.h" + +// standard includes +#include + +// lib includes +extern "C" { +#include +} + +namespace { + /** + * @brief Map FFmpeg's color space to the console overlay's color matrix. + * @param colorspace FFmpeg frame color space. + * @return Supported overlay matrix or a marker requiring software conversion. + */ + streaming::VideoColorMatrix overlay_matrix(AVColorSpace colorspace) { + switch (colorspace) { + case AVCOL_SPC_BT709: + return streaming::VideoColorMatrix::bt709; + case AVCOL_SPC_UNSPECIFIED: + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + return streaming::VideoColorMatrix::bt601; + default: + return streaming::VideoColorMatrix::unsupported; + } + } + + /** + * @brief Validate each positive-stride plane against its FFmpeg-owned allocation. + * @param frame Frame proposed for display. + * @return Read-only view, or an empty view when the layout cannot be displayed safely. + */ + streaming::Yuv420FrameView make_view(const AVFrame &frame) { + if (frame.format != AV_PIX_FMT_YUV420P || frame.width <= 0 || frame.height <= 0) { + return {}; + } + streaming::Yuv420FrameView view; + view.width = frame.width; + view.height = frame.height; + view.matrix = overlay_matrix(frame.colorspace); + view.fullRange = frame.color_range == AVCOL_RANGE_JPEG; + view.interlaced = (frame.flags & AV_FRAME_FLAG_INTERLACED) != 0; + for (std::size_t i = 0; i < view.planes.size(); ++i) { + const int width = i == 0 ? frame.width : frame.width / 2 + frame.width % 2; + const int height = i == 0 ? frame.height : frame.height / 2 + frame.height % 2; + if (frame.data[i] == nullptr || frame.linesize[i] < width) { + return {}; + } + const AVBufferRef *buffer = av_frame_get_plane_buffer(&frame, static_cast(i)); + if (buffer == nullptr) { + return {}; + } + const auto offset = reinterpret_cast(frame.data[i]) - reinterpret_cast(buffer->data); + if (const auto required = static_cast(height - 1) * frame.linesize[i] + width; offset > buffer->size || required > buffer->size - offset) { + return {}; + } + view.planes[i] = frame.data[i]; + view.sizes[i] = buffer->size - offset; + view.pitches[i] = frame.linesize[i]; + } + return view; + } +} // namespace + +namespace streaming { + FfmpegVideoFrame::~FfmpegVideoFrame() { + av_frame_free(&frame_); + } + + bool FfmpegVideoFrame::ensure_frame_allocated() { + if (frame_ == nullptr) { + frame_ = av_frame_alloc(); + } + return frame_ != nullptr; + } + + bool FfmpegVideoFrame::take(AVFrame &source) { + const auto view = make_view(source); + if (view.width == 0) { + return false; + } + if (!ensure_frame_allocated()) { + return false; + } + av_frame_unref(frame_); + av_frame_move_ref(frame_, &source); + view_ = view; + return true; + } + + bool FfmpegVideoFrame::retain(const FfmpegVideoFrame &source) { + if (this == &source) { + return view_.width > 0; + } + if (source.view_.width == 0) { + return false; + } + if (!ensure_frame_allocated()) { + return false; + } + clear(); + if (av_frame_ref(frame_, source.frame_) < 0) { + return false; + } + view_ = source.view_; + return true; + } + + void FfmpegVideoFrame::clear() { + if (frame_ != nullptr) { + av_frame_unref(frame_); + } + view_ = {}; + } + + void FfmpegVideoFrame::swap(FfmpegVideoFrame &other) noexcept { + std::swap(frame_, other.frame_); + std::swap(view_, other.view_); + } + + const Yuv420FrameView &FfmpegVideoFrame::view() const noexcept { + return view_; + } +} // namespace streaming diff --git a/src/streaming/ffmpeg_video_frame.h b/src/streaming/ffmpeg_video_frame.h new file mode 100644 index 0000000..d4d974e --- /dev/null +++ b/src/streaming/ffmpeg_video_frame.h @@ -0,0 +1,67 @@ +/** + * @file src/streaming/ffmpeg_video_frame.h + * @brief Owns decoded FFmpeg planes across publication, presentation and capture. + */ +#pragma once + +// local includes +#include "src/streaming/video_acceleration.h" + +struct AVFrame; + +namespace streaming { + /** + * @brief A reusable slot holding one reference-counted I420 frame and its read-only view. + * + * Each slot has a single thread owner. Holding the reference keeps the planes valid until + * that owner is done, even if the decoder flushes or reuses its receive frame. + */ + class FfmpegVideoFrame { + public: + /** @brief Construct an empty slot. */ + FfmpegVideoFrame() = default; + /** @brief Release the FFmpeg frame and its buffers. */ + ~FfmpegVideoFrame(); + FfmpegVideoFrame(const FfmpegVideoFrame &) = delete; + FfmpegVideoFrame &operator=(const FfmpegVideoFrame &) = delete; + + /** + * @brief Move a validated I420 frame reference from the decoder into this slot. + * @param source The decoder frame, emptied on success. + * @return True when the reference was transferred. + */ + bool take(AVFrame &source); + + /** + * @brief Retain a separate read-only reference for an asynchronous consumer. + * @param source Slot whose pixels should remain available. + * @return True when the reference was acquired. + */ + bool retain(const FfmpegVideoFrame &source); + + /** @brief Release the held buffers while keeping the reusable FFmpeg frame slot. */ + void clear(); + + /** + * @brief Exchange ownership of two slots without touching the pixel buffers. + * @param other Slot to exchange with this one. + */ + void swap(FfmpegVideoFrame &other) noexcept; + + /** + * @brief Read the frame metadata and planes while this reference is held. + * @return Frame view, with zero dimensions when empty. + */ + [[nodiscard]] const Yuv420FrameView &view() const noexcept; + + private: + /** + * @brief Lazily allocate the reusable frame structure. + * @return True when the slot has a frame structure. + */ + bool ensure_frame_allocated(); + + AVFrame *frame_ = nullptr; ///< Lazily allocated frame structure reused on each handoff. + Yuv420FrameView view_ {}; ///< Bounds and metadata for the buffers owned by frame_. + }; +} // namespace streaming diff --git a/src/streaming/session.cpp b/src/streaming/session.cpp index 72e4bd1..14f713e 100644 --- a/src/streaming/session.cpp +++ b/src/streaming/session.cpp @@ -76,8 +76,6 @@ namespace { constexpr Uint32 STREAM_END_STATS_POLL_MILLISECONDS = 50U; constexpr Uint32 STREAM_VIDEO_IDLE_IDR_MILLISECONDS = 1500U; constexpr Uint32 STREAM_VIDEO_IDLE_IDR_COOLDOWN_MILLISECONDS = 2000U; - constexpr int DEFAULT_STREAM_FPS = 30; - constexpr int DEFAULT_STREAM_BITRATE_KBPS = 1000; constexpr int MIN_STREAM_FPS = 15; constexpr int MAX_STREAM_FPS = 60; constexpr int MIN_STREAM_BITRATE_KBPS = 250; @@ -195,8 +193,8 @@ namespace { struct ResolvedStreamParameters { VIDEO_MODE videoMode {}; - int fps = DEFAULT_STREAM_FPS; - int bitrateKbps = DEFAULT_STREAM_BITRATE_KBPS; + int fps = app::DEFAULT_STREAM_FRAMERATE; + int bitrateKbps = app::DEFAULT_STREAM_BITRATE_KBPS; int packetSize = DEFAULT_PACKET_SIZE; int streamingRemotely = STREAM_CFG_AUTO; }; @@ -1507,7 +1505,7 @@ namespace streaming { const VIDEO_MODE &activeStreamVideoMode = scopedStreamVideoMode.active_video_mode(); StreamUiResources resources {}; - resources.statisticsEnabled = settings.showPerformanceStats; + resources.statisticsEnabled = settings.presentation.showPerformanceStats; if (std::string initializationError; !initialize_stream_ui_resources(window, activeStreamVideoMode, preferredController, &resources, &initializationError)) { assign_status_message(statusMessage, initializationError); logging::error("stream", initializationError); @@ -1569,6 +1567,7 @@ namespace streaming { std::string("Requested stream configuration | resolution=") + std::to_string(startContext.streamConfiguration.width) + "x" + std::to_string(startContext.streamConfiguration.height) + " | fps=" + std::to_string(startContext.streamConfiguration.fps) + " | bitrate=" + std::to_string(startContext.streamConfiguration.bitrate) + " kbps" + " | packetSize=" + std::to_string(startContext.streamConfiguration.packetSize) + " | networkProfile=" + describe_streaming_remotely_mode(startContext.streamConfiguration.streamingRemotely) + " | clientRefreshX100=" + std::to_string(startContext.streamConfiguration.clientRefreshRateX100) ); resources.mediaBackend.set_audio_playback_enabled(settings.playAudioOnXbox); + resources.mediaBackend.set_gpu_presentation_enabled(settings.presentation.gpuPresentation); logging::info("stream", std::string("Xbox audio playback ") + (settings.playAudioOnXbox ? "enabled" : "disabled for lower decode latency")); logging::debug( "stream", diff --git a/src/streaming/video_acceleration.cpp b/src/streaming/video_acceleration.cpp index e5b4d5c..f3da8e7 100644 --- a/src/streaming/video_acceleration.cpp +++ b/src/streaming/video_acceleration.cpp @@ -54,15 +54,23 @@ namespace streaming { } std::vector VideoAccelerationReport::summary_lines() const { + const char *presentation = "GPU presentation: no software fallback"; + if (!fallbackReason.empty()) { + presentation = "SOFTWARE FALLBACK USED"; + } else if (gpuFrames == 0 && softwareFrames == 0) { + presentation = "No video frames were presented"; + } else if (!gpuPresentationRequested) { + presentation = "Software presentation selected"; + } std::vector lines { - fallbackReason.empty() ? (gpuFrames == 0 && softwareFrames == 0 ? "No video frames were presented" : "GPU presentation: no software fallback") : "SOFTWARE FALLBACK USED", + presentation, "GPU overlay: " + std::to_string(gpuFrames) + " frames | Software: " + std::to_string(softwareFrames) + " frames", }; if (!fallbackReason.empty()) { lines.push_back("Reason: " + fallbackReason); } lines.push_back("FFmpeg: " + cpuFeatures + (threaded ? " | Threading enabled" : " | THREADING DISABLED")); - lines.emplace_back("Video decoding: CPU (GPU accelerates presentation)"); + lines.emplace_back("Video decoding: CPU"); return lines; } } // namespace streaming diff --git a/src/streaming/video_acceleration.h b/src/streaming/video_acceleration.h index 29de69e..79694d6 100644 --- a/src/streaming/video_acceleration.h +++ b/src/streaming/video_acceleration.h @@ -77,6 +77,7 @@ namespace streaming { std::string fallbackReason; ///< First fallback reason, retained after later successes. std::string cpuFeatures; ///< CPU optimizations enabled in the linked FFmpeg build. bool threaded = false; ///< FFmpeg has internal synchronization, independently of the decoder worker count. + bool gpuPresentationRequested = false; ///< True when the stream was configured to try NV2A presentation. /** * @brief Retain the first reason for using software presentation. diff --git a/src/ui/shell_screen.cpp b/src/ui/shell_screen.cpp index 1033b58..14496ae 100644 --- a/src/ui/shell_screen.cpp +++ b/src/ui/shell_screen.cpp @@ -2507,10 +2507,11 @@ namespace { state.settings.streamFramerate, state.settings.streamBitrateKbps, state.settings.playAudioOnPc, - state.settings.showPerformanceStats, + state.settings.presentation.showPerformanceStats, state.settings.playAudioOnXbox, state.settings.controls.screenshotDirectory, state.settings.controls.keyboardDockTop, + state.settings.presentation.gpuPresentation, }; } diff --git a/tests/unit/app/client_state_test.cpp b/tests/unit/app/client_state_test.cpp index b31bc4c..8105c4f 100644 --- a/tests/unit/app/client_state_test.cpp +++ b/tests/unit/app/client_state_test.cpp @@ -172,6 +172,7 @@ namespace { ASSERT_EQ(state.settings.focusArea, app::SettingsFocusArea::options); } + /** @brief Start on the host toolbar with the default stream profile. */ TEST(ClientStateTest, StartsOnTheHostsScreenWithTheToolbarSelected) { const app::ClientState state = app::create_initial_state(); @@ -182,7 +183,9 @@ namespace { EXPECT_FALSE(state.shell.shouldExit); EXPECT_FALSE(state.hosts.dirty); EXPECT_EQ(state.settings.loggingLevel, logging::LogLevel::none); - EXPECT_EQ(state.settings.streamFramerate, 30); + EXPECT_EQ(state.settings.streamFramerate, 60); + EXPECT_EQ(state.settings.streamBitrateKbps, 3000); + EXPECT_FALSE(state.settings.presentation.gpuPresentation); } TEST(ClientStateTest, ReplacesHostsFromPersistenceWithoutMarkingThemDirty) { @@ -1326,6 +1329,7 @@ namespace { */ TEST(ClientStateTest, DisplaySettingsCanCycleXboxVideoModeStreamChoices) { app::ClientState state = app::create_initial_state(); + state.settings.streamFramerate = 30; state.settings.availableVideoModes = { VIDEO_MODE {640, 480, 32, 60}, VIDEO_MODE {720, 480, 32, 60}, @@ -1362,6 +1366,7 @@ namespace { */ TEST(ClientStateTest, DisplaySettingsPreserveCustomBitrateWithoutAResolutionChange) { app::ClientState state = app::create_initial_state(); + state.settings.streamFramerate = 30; state.settings.availableVideoModes = {VIDEO_MODE {1280, 720, 32, 60}}; state.settings.preferredVideoMode = state.settings.availableVideoModes.front(); state.settings.preferredVideoModeSet = true; @@ -1387,6 +1392,7 @@ namespace { */ TEST(ClientStateTest, DisplaySettingsCanCycleStreamFramerateThroughSixtyFps) { app::ClientState state = app::create_initial_state(); + state.settings.streamFramerate = 30; state.settings.preferredVideoMode = VIDEO_MODE {1280, 720, 32, 50}; state.settings.preferredVideoModeSet = true; state.settings.streamBitrateKbps = 750; @@ -1413,9 +1419,9 @@ namespace { state.settings.streamFramerate = 999; update = app::handle_command(state, input::UiCommand::activate); EXPECT_EQ(update.navigation.activatedItemId, "cycle-stream-framerate"); - EXPECT_EQ(state.settings.streamFramerate, 30); - EXPECT_EQ(state.settings.streamBitrateKbps, 5000); - EXPECT_EQ(state.shell.statusMessage, "Stream frame rate set to 30 FPS"); + EXPECT_EQ(state.settings.streamFramerate, 60); + EXPECT_EQ(state.settings.streamBitrateKbps, 10000); + EXPECT_EQ(state.shell.statusMessage, "Stream frame rate set to 60 FPS"); } /** @@ -1423,6 +1429,7 @@ namespace { */ TEST(ClientStateTest, DisplaySettingsUseDefaultResolutionForFramerateChangesBeforeDetection) { app::ClientState state = app::create_initial_state(); + state.settings.streamFramerate = 30; ASSERT_NO_FATAL_FAILURE(open_display_settings_options(state)); ASSERT_TRUE(state.detailMenu.select_item_by_id("cycle-stream-framerate")); @@ -1478,10 +1485,36 @@ namespace { } } + /** @brief Switch from the software default to GPU and back while preserving bitrate and statistics. */ + TEST(ClientStateTest, DisplaySettingsCanSelectGpuPresentation) { + auto state = app::create_initial_state(); + ASSERT_NO_FATAL_FAILURE(open_display_settings_options(state)); + ASSERT_TRUE(state.detailMenu.select_item_by_id("toggle-gpu-presentation")); + ASSERT_FALSE(state.settings.presentation.gpuPresentation); + state.settings.streamBitrateKbps = 4500; + state.settings.presentation.showPerformanceStats = true; + + EXPECT_TRUE(app::handle_command(state, input::UiCommand::activate).persistence.settingsChanged); + EXPECT_TRUE(state.settings.presentation.gpuPresentation); + EXPECT_TRUE(state.settings.dirty); + ASSERT_NE(state.detailMenu.selected_item(), nullptr); + EXPECT_EQ(state.detailMenu.selected_item()->label, "Video Presentation: GPU"); + EXPECT_EQ(state.settings.streamBitrateKbps, 4500); + EXPECT_TRUE(state.settings.presentation.showPerformanceStats); + + EXPECT_TRUE(app::handle_command(state, input::UiCommand::activate).persistence.settingsChanged); + EXPECT_FALSE(state.settings.presentation.gpuPresentation); + EXPECT_TRUE(state.settings.presentation.showPerformanceStats); + ASSERT_NE(state.detailMenu.selected_item(), nullptr); + EXPECT_EQ(state.detailMenu.selected_item()->label, "Video Presentation: Software"); + } + + /** @brief Toggle audio and statistics independently while retaining the selected presentation path. */ TEST(ClientStateTest, DisplaySettingsCanToggleXboxAudioAndPerformanceStats) { app::ClientState state = app::create_initial_state(); ASSERT_TRUE(state.settings.playAudioOnXbox); - ASSERT_FALSE(state.settings.showPerformanceStats); + ASSERT_FALSE(state.settings.presentation.showPerformanceStats); + state.settings.presentation.gpuPresentation = true; ASSERT_NO_FATAL_FAILURE(open_display_settings_options(state)); @@ -1498,7 +1531,8 @@ namespace { update = app::handle_command(state, input::UiCommand::activate); EXPECT_EQ(update.navigation.activatedItemId, "toggle-show-performance-stats"); EXPECT_TRUE(update.persistence.settingsChanged); - EXPECT_TRUE(state.settings.showPerformanceStats); + EXPECT_TRUE(state.settings.presentation.showPerformanceStats); + EXPECT_TRUE(state.settings.presentation.gpuPresentation); EXPECT_EQ(state.shell.statusMessage, "Performance stats enabled"); ASSERT_NE(state.detailMenu.selected_item(), nullptr); EXPECT_EQ(state.detailMenu.selected_item()->label, "Performance Stats: On"); diff --git a/tests/unit/app/directory_picker_test.cpp b/tests/unit/app/directory_picker_test.cpp index 7bfd044..2b8915b 100644 --- a/tests/unit/app/directory_picker_test.cpp +++ b/tests/unit/app/directory_picker_test.cpp @@ -257,8 +257,8 @@ TEST(DirectoryPickerTest, KeepsStorageSeparateFromTheExistingDisplayCategory) { app::handle_command(state, input::UiCommand::move_up); app::handle_command(state, input::UiCommand::activate); EXPECT_EQ(state.settings.selectedCategory, app::SettingsCategory::display); - EXPECT_EQ(state.detailMenu.items().size(), 6); EXPECT_EQ(state.detailMenu.selected_item()->id, "cycle-stream-video-mode"); + EXPECT_TRUE(state.detailMenu.select_item_by_id("toggle-gpu-presentation")); EXPECT_FALSE(state.detailMenu.select_item_by_id("screenshot-directory")); EXPECT_EQ(state.settings.controls.screenshotDirectory, "F:\\Screenshots"); } diff --git a/tests/unit/app/settings_storage_test.cpp b/tests/unit/app/settings_storage_test.cpp index bf8fd24..8db2074 100644 --- a/tests/unit/app/settings_storage_test.cpp +++ b/tests/unit/app/settings_storage_test.cpp @@ -107,6 +107,36 @@ namespace { EXPECT_FALSE(loadResult.settings.playAudioOnXbox); } + /** @brief Persist explicit GPU selection and default to software with old or invalid settings. */ + TEST_F(SettingsStorageTest, SavesAndLoadsPresentationPreference) { + write_text_file(settingsPath, "[streaming]\nfps = 30\n"); + const auto partial = app::load_app_settings(settingsPath); + EXPECT_FALSE(partial.settings.gpuPresentation); + EXPECT_EQ(partial.settings.streamFramerate, 30); + EXPECT_EQ(partial.settings.streamBitrateKbps, 1000); + + app::AppSettings settings; + settings.gpuPresentation = true; + ASSERT_TRUE(app::save_app_settings(settings, settingsPath).success); + const auto gpu = app::load_app_settings(settingsPath); + EXPECT_TRUE(gpu.settings.gpuPresentation); + EXPECT_TRUE(gpu.warnings.empty()); + EXPECT_FALSE(gpu.cleanupRequired); + + write_text_file(settingsPath, "[streaming]\ngpu_presentation = 'software'\n"); + const auto invalid = app::load_app_settings(settingsPath); + EXPECT_FALSE(invalid.settings.gpuPresentation); + ASSERT_EQ(invalid.warnings.size(), 1U); + EXPECT_NE(invalid.warnings.front().find("streaming.gpu_presentation"), std::string::npos); + } + + /** @brief A partial settings file derives bitrate from the saved resolution and active frame rate. */ + TEST_F(SettingsStorageTest, DerivesMissingBitrateFromSavedStreamMode) { + write_text_file(settingsPath, "[streaming]\nvideo_width = 1280\nvideo_height = 720\nvideo_mode_selected = true\n"); + EXPECT_EQ(app::load_app_settings(settingsPath).settings.streamBitrateKbps, 10000); + } + + /** @brief Missing settings leave the default stream profile and produce no warnings. */ TEST_F(SettingsStorageTest, MissingFilesReturnDefaultsWithoutWarnings) { const app::LoadAppSettingsResult loadResult = app::load_app_settings(settingsPath); @@ -117,13 +147,15 @@ namespace { EXPECT_EQ(loadResult.settings.xemuConsoleLoggingLevel, logging::LogLevel::none); EXPECT_EQ(loadResult.settings.logViewerPlacement, app::LogViewerPlacement::full); EXPECT_FALSE(loadResult.settings.preferredVideoModeSet); - EXPECT_EQ(loadResult.settings.streamFramerate, 30); - EXPECT_EQ(loadResult.settings.streamBitrateKbps, 1000); + EXPECT_EQ(loadResult.settings.streamFramerate, 60); + EXPECT_EQ(loadResult.settings.streamBitrateKbps, 3000); + EXPECT_FALSE(loadResult.settings.gpuPresentation); EXPECT_FALSE(loadResult.settings.playAudioOnPc); EXPECT_FALSE(loadResult.settings.showPerformanceStats); EXPECT_TRUE(loadResult.settings.playAudioOnXbox); } + /** @brief Invalid saved preferences produce warnings and fall back to the current defaults. */ TEST_F(SettingsStorageTest, InvalidValuesFallBackToDefaultsWithWarnings) { write_text_file( settingsPath, @@ -149,7 +181,7 @@ namespace { EXPECT_EQ(loadResult.settings.xemuConsoleLoggingLevel, logging::LogLevel::none); EXPECT_EQ(loadResult.settings.logViewerPlacement, app::LogViewerPlacement::full); EXPECT_FALSE(loadResult.settings.preferredVideoModeSet); - EXPECT_EQ(loadResult.settings.streamFramerate, 30); + EXPECT_EQ(loadResult.settings.streamFramerate, 60); EXPECT_FALSE(loadResult.settings.playAudioOnPc); EXPECT_TRUE(loadResult.settings.playAudioOnXbox); } diff --git a/tests/unit/streaming/video_acceleration_test.cpp b/tests/unit/streaming/video_acceleration_test.cpp index 3162cf0..836f1e7 100644 --- a/tests/unit/streaming/video_acceleration_test.cpp +++ b/tests/unit/streaming/video_acceleration_test.cpp @@ -95,7 +95,21 @@ TEST(VideoAccelerationTest, RetainsFirstFallbackAfterAccelerationRecovers) { } /** - * @brief Published plane buffers retain alignment and zero padding after reuse and moves. + * @brief Explicit software rendering is distinct from a failed GPU attempt in the summary. + */ +TEST(VideoAccelerationTest, ReportsSelectedSoftwareWithoutAFallback) { + streaming::VideoAccelerationReport report; + report.gpuPresentationRequested = false; + report.softwareFrames = 3; + const auto lines = report.summary_lines(); + ASSERT_GE(lines.size(), 2U); + EXPECT_EQ(lines[0], "Software presentation selected"); + EXPECT_EQ(lines[1], "GPU overlay: 0 frames | Software: 3 frames"); + EXPECT_TRUE(report.fallbackReason.empty()); +} + +/** + * @brief Media scratch buffers retain alignment and zero padding after reuse and moves. */ TEST(VideoAccelerationTest, KeepsMediaBuffersAlignedAndPadded) { streaming::AlignedMediaBuffer buffer;