Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,10 +74,9 @@ No network connection or JavaScript runtime is needed to display the icons on th

## Video presentation

**Settings → Display → Video Presentation** defaults to **Software**, which writes directly to the
framebuffer when available and performed better in xemu. **GPU** uses NV2A color conversion and scaling
with automatic software fallback. The setting is saved as `streaming.gpu_presentation` and both paths
are available in the same build.
**Settings → Display → Video Presentation** defaults to **GPU**, which uses NV2A color conversion and
scaling with automatic software fallback. **Software** combines the decoded video and overlay on the
CPU before displaying each frame. Moonlight keeps your selection in `streaming.gpu_presentation`.

## Stream audio

Expand Down
2 changes: 1 addition & 1 deletion src/_nxdk_compat/ffmpeg_memory.h
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/**
* @file src/_nxdk_compat/ffmpeg_memory.h
* @brief Declares the aligned allocation family used privately by FFmpeg.
* @brief Declares aligned allocation for FFmpeg and software scanout on Xbox.
*/
#pragma once

Expand Down
3 changes: 2 additions & 1 deletion src/app/client_state.h
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ namespace app {

inline constexpr int DEFAULT_STREAM_FRAMERATE = 60; ///< Stream frame rate at the default 640x480 resolution.
inline constexpr int DEFAULT_STREAM_BITRATE_KBPS = 3000; ///< Bitrate for the default 640x480, 60 FPS profile.
inline constexpr bool DEFAULT_GPU_PRESENTATION = true; ///< Default to NV2A presentation with software fallback.

/**
* @brief Focus areas used by the two-pane settings screen.
Expand Down Expand Up @@ -387,7 +388,7 @@ namespace app {
* @brief Preferences for presenting decoded video and its on-screen statistics.
*/
struct VideoPresentationPreferences {
bool gpuPresentation = false; ///< Opt in to NV2A presentation with software fallback on new streams.
bool gpuPresentation = DEFAULT_GPU_PRESENTATION; ///< Use NV2A presentation with software fallback on new streams.
bool showPerformanceStats = false; ///< Display live performance telemetry and include it in the stream summary.
};

Expand Down
4 changes: 2 additions & 2 deletions src/app/settings_storage.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ namespace app {
bool playAudioOnXbox = true; ///< True when the Xbox should decode and play streamed audio locally.
std::string screenshotDirectory; ///< Screenshot folder; empty selects the title-scoped screenshots directory.
bool keyboardDockTop = false; ///< Dock the stream keyboard to the top instead of the bottom.
bool gpuPresentation = false; ///< Opt in to NV2A presentation with software fallback.
bool gpuPresentation = DEFAULT_GPU_PRESENTATION; ///< Use NV2A presentation with software fallback.
streaming::AudioMode audioMode = streaming::AudioMode::stereo; ///< Source channels and local playback mix.
};

Expand Down Expand Up @@ -59,7 +59,7 @@ namespace app {
std::string default_settings_path();

/**
* @brief Load saved preferences, defaulting to stereo and deriving the bitrate when absent.
* @brief Load saved preferences. Missing values use GPU presentation, stereo audio, and a bitrate derived from the stream mode.
*
* @param filePath Settings file to read.
* @return Loaded settings plus any non-fatal warnings.
Expand Down
110 changes: 59 additions & 51 deletions src/streaming/ffmpeg_stream_backend.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -595,17 +595,6 @@ namespace streaming {
return video_.codecName;
}

/**
* @brief Return whether two SDL rectangles describe the same area.
*
* @param left First rectangle.
* @param right Second rectangle.
* @return True when every rectangle field matches.
*/
bool rectangles_match(const SDL_Rect &left, const SDL_Rect &right) {
return left.x == right.x && left.y == right.y && left.w == right.w && left.h == right.h;
}

#ifdef NXDK
/**
* @brief Convert the active Xbox framebuffer bpp to FFmpeg's packed RGB format.
Expand Down Expand Up @@ -663,10 +652,10 @@ namespace streaming {
" ms / age " + std::to_string(video_.lastReceiveAgeUs.load() / 1000U) + " ms | " + audio_.describe_playback();
}

bool FfmpegStreamBackend::render_latest_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer) {
bool FfmpegStreamBackend::render_latest_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight) {
const auto before = acceleration_.gpuFrames + acceleration_.softwareFrames;
const auto started = PltGetMicroseconds();
const bool result = render_video_frame(renderer, screenWidth, screenHeight, allowDirectFramebuffer);
const bool result = render_video_frame(renderer, screenWidth, screenHeight);
if (result) {
statisticsBitmapChanged_ = false;
if (acceleration_.gpuFrames + acceleration_.softwareFrames > before) {
Expand All @@ -676,12 +665,12 @@ namespace streaming {
return result;
}

bool FfmpegStreamBackend::render_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer) {
bool FfmpegStreamBackend::render_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight) {
if (renderer == nullptr || !video_.hasFrame.load()) {
return false;
}

bool textureNeedsUpload = false;
bool newFrame = false;
if (const std::uint64_t publishedFrameVersion = video_.publishedFrameVersion.load(); publishedFrameVersion != video_.renderedFrameVersion) {
std::scoped_lock lock(video_.frame_mutex());
if (video_.latestFrameVersion != video_.renderedFrameVersion && video_.latestFrame.view().width > 0) {
Expand All @@ -690,7 +679,7 @@ namespace streaming {
}
video_.renderFrame.swap(video_.latestFrame);
video_.renderedFrameVersion = video_.latestFrameVersion;
textureNeedsUpload = true;
newFrame = true;
}
}

Expand All @@ -699,40 +688,56 @@ namespace streaming {
return false;
}

const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, frame.width, frame.height);
#ifdef NXDK
if (allowDirectFramebuffer && gpuPresentationEnabled_ && !overlayFailed_) {
if (!textureNeedsUpload && !statisticsBitmapChanged_ && presenter_.active()) {
return true;
}
const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, frame.width, frame.height);
if (!presenter_.active()) {
const VIDEO_MODE mode = XVideoGetMode();
if (auto *framebuffer = XVideoGetFB(); framebuffer != nullptr && mode.width > 0 && mode.height > 0 && (mode.bpp == 16 || mode.bpp == 32)) {
std::memset(framebuffer, 0, static_cast<std::size_t>(mode.width) * static_cast<std::size_t>(mode.height) * static_cast<std::size_t>(mode.bpp / 8));
XVideoFlushFB();
}
}
std::string reason;
if (presenter_.present(frame, {destination.x, destination.y, destination.w, destination.h}, &reason, {}, &statisticsOverlay_)) {
if (textureNeedsUpload) {
++acceleration_.gpuFrames;
if (render_gpu_video_frame(destination, newFrame)) {
return true;
}
if (!softwareScanoutFailed_) {
if (render_latest_video_frame_to_framebuffer(screenWidth, screenHeight)) {
if (newFrame) {
++acceleration_.softwareFrames;
}
return true;
}
softwareScanoutFailed_ = true;
logging::warn("stream", "Double-buffered scanout unavailable; using SDL software presentation");
stop_video_presentation();
}
#endif
return render_sdl_video_frame(renderer, destination, newFrame);
}

#ifdef NXDK
bool FfmpegStreamBackend::render_gpu_video_frame(const SDL_Rect &destination, bool newFrame) {
if (!gpuPresentationEnabled_ || overlayFailed_) {
return false;
}
if (!newFrame && !statisticsBitmapChanged_ && presenter_.active()) {
return true;
}
if (!presenter_.active()) {
const VIDEO_MODE mode = XVideoGetMode();
if (auto *framebuffer = XVideoGetFB(); framebuffer != nullptr && mode.width > 0 && mode.height > 0 && (mode.bpp == 16 || mode.bpp == 32)) {
std::memset(framebuffer, 0, static_cast<std::size_t>(mode.width) * static_cast<std::size_t>(mode.height) * static_cast<std::size_t>(mode.bpp / 8));
XVideoFlushFB();
}
}
if (std::string reason; !presenter_.present(video_.renderFrame.view(), {destination.x, destination.y, destination.w, destination.h}, &reason, {}, &statisticsOverlay_)) {
overlayFailed_ = true;
acceleration_.record_fallback(reason);
logging::warn("stream", "GPU presentation fallback: " + reason);
return false;
}
if (allowDirectFramebuffer && render_latest_video_frame_to_framebuffer(screenWidth, screenHeight)) {
if (textureNeedsUpload) {
++acceleration_.softwareFrames;
}
return true;
if (newFrame) {
++acceleration_.gpuFrames;
}
#else
(void) allowDirectFramebuffer;
return true;
}
#endif

bool FfmpegStreamBackend::render_sdl_video_frame(SDL_Renderer *renderer, const SDL_Rect &destination, bool textureNeedsUpload) {
const auto &frame = video_.renderFrame.view();
if (video_.texture == nullptr || video_.textureWidth != frame.width || video_.textureHeight != frame.height) {
if (video_.texture != nullptr) {
SDL_DestroyTexture(video_.texture);
Expand All @@ -755,10 +760,12 @@ namespace streaming {
return false;
}

const SDL_Rect destination = build_letterboxed_destination(screenWidth, screenHeight, frame.width, frame.height);
SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
SDL_RenderClear(renderer);
const bool presented = SDL_RenderCopy(renderer, video_.texture, nullptr, &destination) == 0;
if (presented) {
render_sdl_statistics(renderer, destination);
SDL_RenderPresent(renderer);
}
if (presented && textureNeedsUpload) {
++acceleration_.softwareFrames;
Expand Down Expand Up @@ -808,6 +815,7 @@ namespace streaming {
acceleration_.gpuPresentationRequested = gpuPresentationEnabled_;
statistics_.reset();
overlayFailed_ = false;
softwareScanoutFailed_ = false;
const int cpuFlags = av_get_cpu_flags();
acceleration_.cpuFeatures = (cpuFlags & AV_CPU_FLAG_SSE) != 0 ? "MMX/SSE" : ((cpuFlags & AV_CPU_FLAG_MMX) != 0 ? "MMX" : "Scalar");
acceleration_.threaded = std::string(avcodec_configuration()).find("--enable-w32threads") != std::string::npos;
Expand Down Expand Up @@ -885,6 +893,9 @@ namespace streaming {
acceleration_.record_fallback("GPU overlay teardown timed out; acceleration disabled");
logging::warn("stream", "GPU overlay teardown timed out; buffers retained to avoid DMA into freed storage");
}
if (!softwareScanout_.reset()) {
logging::warn("stream", "Software scanout retirement timed out; extra framebuffer retained");
}
}

void FfmpegStreamBackend::cleanup_video_decoder() {
Expand Down Expand Up @@ -935,8 +946,6 @@ namespace streaming {
video_.renderedFrameVersion = 0;
video_.publishedFrameVersion.store(0);
video_.codecName = "video";
video_.directFramebufferDestination = SDL_Rect {0, 0, 0, 0};
video_.directFramebufferCleared = false;
video_.decoderStopRequested.store(false);
video_.hasFrame.store(false);
video_.submittedDecodeUnitCount.store(0);
Expand Down Expand Up @@ -1071,10 +1080,9 @@ namespace streaming {
bool FfmpegStreamBackend::render_latest_video_frame_to_framebuffer(int screenWidth, int screenHeight) {
#ifdef NXDK
const VIDEO_MODE videoMode = XVideoGetMode();
std::uint8_t *framebuffer = XVideoGetFB();
const int bytesPerPixel = xbox_framebuffer_bytes_per_pixel(videoMode.bpp);
const AVPixelFormat destinationFormat = xbox_framebuffer_pixel_format(videoMode.bpp);
if (framebuffer == nullptr || videoMode.width <= 0 || videoMode.height <= 0 || bytesPerPixel <= 0 || destinationFormat == AV_PIX_FMT_NONE) {
if (videoMode.width <= 0 || videoMode.height <= 0 || bytesPerPixel <= 0 || destinationFormat == AV_PIX_FMT_NONE) {
return false;
}

Expand All @@ -1087,10 +1095,11 @@ namespace streaming {
}

const int framebufferPitch = videoMode.width * bytesPerPixel;
if (!video_.directFramebufferCleared || !rectangles_match(video_.directFramebufferDestination, destination)) {
std::memset(framebuffer, 0, static_cast<std::size_t>(framebufferPitch) * static_cast<std::size_t>(videoMode.height));
video_.directFramebufferDestination = destination;
video_.directFramebufferCleared = true;
const auto framebufferSize = static_cast<std::size_t>(framebufferPitch) * videoMode.height;
const VideoRectangle videoArea {destination.x, destination.y, destination.w, destination.h};
auto *framebuffer = softwareScanout_.back_buffer(framebufferSize, videoArea);
if (framebuffer == nullptr) {
return false;
}

video_.presentScaleContext = sws_getCachedContext(
Expand Down Expand Up @@ -1141,9 +1150,8 @@ namespace streaming {
return false;
}

statisticsOverlay_.draw_rgb(framebuffer, static_cast<std::size_t>(framebufferPitch) * videoMode.height, framebufferPitch, videoMode.width, videoMode.height, videoMode.bpp, {destination.x, destination.y, destination.w, destination.h});
XVideoFlushFB();
return true;
statisticsOverlay_.draw_rgb(framebuffer, framebufferSize, framebufferPitch, videoMode.width, videoMode.height, videoMode.bpp, videoArea);
return softwareScanout_.present();
#else
(void) screenWidth;
(void) screenHeight;
Expand Down
30 changes: 22 additions & 8 deletions src/streaming/ffmpeg_stream_backend.h
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include "src/streaming/audio_player.h"
#include "src/streaming/ffmpeg_video_frame.h"
#include "src/streaming/pvideo_presenter.h"
#include "src/streaming/software_scanout.h"
#include "src/streaming/stats_overlay.h"

struct AVCodecContext;
Expand Down Expand Up @@ -183,10 +184,9 @@ namespace streaming {
* @param renderer SDL renderer used by the stream session.
* @param screenWidth Current renderer output width.
* @param screenHeight Current renderer output height.
* @param allowDirectFramebuffer True when platform-specific direct presentation may bypass SDL's renderer.
* @return True when a decoded frame was available and rendered.
* @return True when decoded video is displayed on any presentation path.
*/
bool render_latest_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer = true);
bool render_latest_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight);

/**
* @brief Restore CPU framebuffer output before drawing shutdown or summary UI on the render thread.
Expand Down Expand Up @@ -262,10 +262,24 @@ namespace streaming {
* @param renderer SDL video renderer.
* @param screenWidth Output width.
* @param screenHeight Output height.
* @param allowDirectFramebuffer Permit Xbox GPU or direct framebuffer presentation.
* @return True after successful presentation.
*/
bool render_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight, bool allowDirectFramebuffer);
bool render_video_frame(SDL_Renderer *renderer, int screenWidth, int screenHeight);
/**
* @brief Present the current video and UI through the GPU overlay when available.
* @param destination Output video rectangle.
* @param newFrame True when a new decoded frame was acquired.
* @return True when GPU presentation is active and the output is up to date.
*/
bool render_gpu_video_frame(const SDL_Rect &destination, bool newFrame);
/**
* @brief Upload and present the current video and UI through SDL.
* @param renderer SDL video renderer.
* @param destination Output video rectangle.
* @param textureNeedsUpload True when a new decoded frame was acquired.
* @return True after successful SDL presentation.
*/
bool render_sdl_video_frame(SDL_Renderer *renderer, const SDL_Rect &destination, bool textureNeedsUpload);
/**
* @brief Draw the cached panel when the SDL presentation fallback is used.
* @param renderer SDL video renderer.
Expand Down Expand Up @@ -406,11 +420,9 @@ namespace streaming {
mutable std::mutex frameMutex_; ///< Guards latest decoded frame publication between worker and render threads.

public:
SDL_Rect directFramebufferDestination {0, 0, 0, 0};
FfmpegVideoFrame latestFrame; ///< Mailbox slot exchanged under frameMutex_.
FfmpegVideoFrame decodeFrame; ///< Decoder-owned slot reused to publish each new reference.
FfmpegVideoFrame renderFrame; ///< Render-thread reference retained for presentation and UI redraws.
bool directFramebufferCleared = false;
std::atomic<bool> decoderStopRequested = false;
std::atomic<bool> hasFrame = false;
std::atomic<std::uint64_t> publishedFrameVersion = 0;
Expand All @@ -427,15 +439,17 @@ namespace streaming {
AudioPlayer audio_; ///< Session Opus decoder and SDL playback device.
ScreenshotTask screenshot_ {}; ///< Single capture slot, bounded to one additional YUV frame.
PvideoPresenter presenter_; ///< Main-thread GPU presentation resources.
SoftwareScanout softwareScanout_; ///< Main-thread software scanout pages.
VideoOverlay statisticsOverlay_; ///< Shared cached panel for all presentation paths.
VideoStatistics statistics_; ///< Cumulative decode and presentation telemetry.
SDL_Texture *statisticsTexture_ = nullptr; ///< Cached panel texture for SDL's final fallback.
bool statisticsBitmapChanged_ = false; ///< A panel update requires refreshing a paused video frame.
bool statisticsTextureChanged_ = false; ///< SDL texture requires the current bitmap.
VideoRectangle statisticsTextureDamage_ {}; ///< Pending SDL texture upload, accumulated until software presentation.
VideoAccelerationReport acceleration_; ///< Session diagnostics survive decoder cleanup.
bool gpuPresentationEnabled_ = false; ///< Session opt-in applied before decoder setup.
bool gpuPresentationEnabled_ = false; ///< Session preference applied before decoder setup.
bool overlayFailed_ = false; ///< Avoid repeating a known failing hardware path this session.
bool softwareScanoutFailed_ = false; ///< Use the SDL fallback after software scanout fails.
};

} // namespace streaming
Loading
Loading