From 73925194a22ac5330993561a4072a0607fd4a14e Mon Sep 17 00:00:00 2001 From: luis henrique Date: Sat, 21 Nov 2026 00:45:11 -0300 Subject: [PATCH 1/4] fix(media-logic): stop degrading render quality on decoder-bound drops Dropped video frames lowered the quality level even when GPU time and compositor pacing were well inside budget. On Quest 3 an 8192x4096 HEVC Main 10 file at ~122 Mbps decoded at ~47 of 60 fps with the GPU at ~1.9 ms per frame; the governor fell from HIGH to EMERGENCY in 6 s, cutting render scale to 0.7 and refresh to 72 Hz without helping the decoder. Drops now count as render stress only when render is not healthy, and no longer block recovery once it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DKhrqTrv7b11FgaPF1e7FV --- rust/media-logic/src/quality.rs | 70 +++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 11 deletions(-) diff --git a/rust/media-logic/src/quality.rs b/rust/media-logic/src/quality.rs index a378617..32d2797 100644 --- a/rust/media-logic/src/quality.rs +++ b/rust/media-logic/src/quality.rs @@ -341,8 +341,15 @@ impl QualityController { return QualityAction::Degrade(self.current_level, self.last_reason); } + // Video frames are also dropped when the hardware decoder cannot keep up with the + // stream's bitrate. With GPU time and compositor pacing both inside their healthy + // budgets the renderer is not the cause, and lowering render scale or refresh rate + // would only cost image quality, so drops count against rendering only otherwise. + let render_has_headroom = sample.frame_time_ms < pacing_healthy_ms + && sample.smoothed_gpu_time_ms < gpu_healthy_ms; + // 2. Verificação de Descarte Severo de Frames: - if sample.dropped_fps >= Self::DROPPED_FPS_STRESS_THRESHOLD { + if sample.dropped_fps >= Self::DROPPED_FPS_STRESS_THRESHOLD && !render_has_headroom { self.dropped_fps_stress_count += 1; self.stable_healthy_samples = 0; if self.dropped_fps_stress_count >= Self::DROPPED_FPS_STRESS_THRESHOLD_SAMPLES @@ -400,10 +407,7 @@ impl QualityController { QualityLevel::High | QualityLevel::Ultra => sample.thermal_level <= 1, }; - let is_healthy = is_thermal_permitting_upgrade - && sample.dropped_fps < 0.5f32 - && sample.frame_time_ms < pacing_healthy_ms - && sample.smoothed_gpu_time_ms < gpu_healthy_ms; + let is_healthy = is_thermal_permitting_upgrade && render_has_headroom; if is_healthy { self.stable_healthy_samples += 1; @@ -416,10 +420,7 @@ impl QualityController { self.reset_counters(); return QualityAction::Upgrade(self.current_level, self.last_reason); } - } else if sample.smoothed_gpu_time_ms > gpu_healthy_ms - || sample.frame_time_ms > pacing_healthy_ms - || sample.dropped_fps >= 0.5f32 - { + } else if !render_has_headroom { // Condição marginal ou estresse: reinicia o acúmulo de estabilidade para evitar flapping self.stable_healthy_samples = 0; } @@ -594,9 +595,10 @@ mod tests { let mut controller = QualityController::new(); assert_eq!(controller.current_level(), QualityLevel::High); + // Marginal GPU (above gpu_healthy_ms, below gpu_budget_ms): drops count as render stress. let sample_drop = QualitySample { thermal_level: 0, - smoothed_gpu_time_ms: 6.0, + smoothed_gpu_time_ms: 8.0, frame_time_ms: 11.0, dropped_fps: 4.5, target_fps: 90.0, @@ -633,6 +635,7 @@ mod tests { target_fps: 90.0, }; let sample_spike = QualitySample { + smoothed_gpu_time_ms: 8.0, dropped_fps: 56.0, ..sample_healthy }; @@ -658,7 +661,7 @@ mod tests { let mut controller = QualityController::new(); let sample_drop = QualitySample { thermal_level: 0, - smoothed_gpu_time_ms: 6.0, + smoothed_gpu_time_ms: 8.0, frame_time_ms: 11.0, dropped_fps: 5.0, target_fps: 90.0, @@ -922,4 +925,49 @@ mod tests { assert_eq!(qc.target_refresh_rate(QualityLevel::Low), 72.0); assert_eq!(qc.target_refresh_rate(QualityLevel::Emergency), 72.0); } + + // Telemetry of an 8192x4096 HEVC Main 10 file at ~122 Mbps on Quest 3: the hardware decoder + // delivered ~47 of 60 fps while the GPU took ~1.9 ms of an 11.1 ms frame. Lowering render + // scale or refresh rate cannot help a decoder-bound stream. + fn decode_bound_sample() -> QualitySample { + QualitySample { + thermal_level: 0, + smoothed_gpu_time_ms: 1.9, + frame_time_ms: 1000.0 / 90.0, + dropped_fps: 11.0, + target_fps: 90.0, + } + } + + #[test] + fn test_dropped_frames_with_render_headroom_do_not_degrade() { + let mut controller = QualityController::new(); + + for i in 0..20 { + assert_eq!( + controller.evaluate(&decode_bound_sample()), + QualityAction::Maintain, + "sample {i}: decoder-bound drops must not lower render quality" + ); + } + assert_eq!(controller.current_level(), QualityLevel::High); + } + + #[test] + fn test_dropped_frames_with_render_headroom_do_not_block_recovery() { + let mut controller = QualityController::new(); + controller.set_manual_level(QualityLevel::Medium); + + let actions: Vec<_> = (0..QualityController::UPGRADE_STABILITY_REQUIRED_SAMPLES) + .map(|_| controller.evaluate(&decode_bound_sample())) + .collect(); + + assert_eq!( + actions.last(), + Some(&QualityAction::Upgrade( + QualityLevel::High, + QualityTransitionReason::StableRecovery + )) + ); + } } From 587cc5472013123124a1c0aab03d3520279df84b Mon Sep 17 00:00:00 2001 From: luis henrique Date: Sat, 21 Nov 2026 00:45:49 -0300 Subject: [PATCH 2/4] fix(bridge): reset the quality level when a new video starts The quality controller is process-global and kept the level earned by the previous file. After a decoder-bound 8K clip a 4320x2160 clip started in EMERGENCY (72 Hz, render scale 0.7) and needed ~58 s of 30-sample recovery steps to reach MEDIUM. Every playback entry point now resets the level, reason and stress counters, unless adaptive mode is off and the level is the user's choice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DKhrqTrv7b11FgaPF1e7FV --- rust/bridge/src/lib.rs | 18 +++++++++++++ rust/media-logic/src/quality.rs | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/rust/bridge/src/lib.rs b/rust/bridge/src/lib.rs index eaddd60..c1090d5 100644 --- a/rust/bridge/src/lib.rs +++ b/rust/bridge/src/lib.rs @@ -506,6 +506,12 @@ pub extern "C" fn evaluate_upscaling_ffi( static QUALITY_CONTROLLER: Lazy> = Lazy::new(|| Mutex::new(media_logic::quality::QualityController::new())); +fn reset_quality_for_new_video() { + if let Ok(mut qc) = QUALITY_CONTROLLER.lock() { + qc.reset_for_new_content(); + } +} + #[no_mangle] pub extern "C" fn quality_controller_set_enabled(enabled: u32) { if let Ok(mut qc) = QUALITY_CONTROLLER.lock() { @@ -1323,6 +1329,8 @@ pub extern "C" fn start_smb_playback( } reset_3d_mode(); + + reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { controller.stop(); @@ -1535,6 +1543,8 @@ pub extern "C" fn start_ftp_playback( } reset_3d_mode(); + + reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { controller.stop(); @@ -1701,6 +1711,8 @@ pub extern "C" fn start_sftp_playback( } reset_3d_mode(); + + reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { controller.stop(); @@ -1920,6 +1932,8 @@ pub extern "C" fn start_nfs_playback( } reset_3d_mode(); + + reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { controller.stop(); @@ -2098,6 +2112,8 @@ pub extern "C" fn start_webdav_playback( } reset_3d_mode(); + + reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { controller.stop(); @@ -2430,6 +2446,8 @@ pub extern "C" fn start_video_playback(path: *const std::os::raw::c_char, start_ } reset_3d_mode(); + + reset_quality_for_new_video(); if path_str.starts_with("http://") || path_str.starts_with("https://") { DOWNLOAD_MANAGER.set_playback_active(true); } diff --git a/rust/media-logic/src/quality.rs b/rust/media-logic/src/quality.rs index 32d2797..5c2bef8 100644 --- a/rust/media-logic/src/quality.rs +++ b/rust/media-logic/src/quality.rs @@ -296,6 +296,18 @@ impl QualityController { self.stable_healthy_samples = 0; } + /// Starts a new video from the default level. A degradation earned by the previous file + /// says nothing about the next one, which would otherwise spend ~30 s per level recovering. + /// With adaptive mode off the level is the user's choice and stays. + pub fn reset_for_new_content(&mut self) { + if !self.enabled { + return; + } + self.current_level = QualityLevel::High; + self.last_reason = QualityTransitionReason::None; + self.reset_counters(); + } + /// Avalia a amostra de performance e atualiza o estado interno do controlador. pub fn evaluate(&mut self, sample: &QualitySample) -> QualityAction { if !self.enabled { @@ -970,4 +982,38 @@ mod tests { )) ); } + + #[test] + fn test_reset_for_new_content_drops_previous_video_degradation() { + let mut controller = QualityController::new(); + let gpu_overload = QualitySample { + thermal_level: 0, + smoothed_gpu_time_ms: 10.5, + frame_time_ms: 1000.0 / 90.0, + dropped_fps: 0.0, + target_fps: 90.0, + }; + controller.evaluate(&gpu_overload); + controller.evaluate(&gpu_overload); + assert_eq!(controller.current_level(), QualityLevel::Medium); + // One stress sample pending: without a counter reset, the next one would degrade at once. + controller.evaluate(&gpu_overload); + + controller.reset_for_new_content(); + + assert_eq!(controller.current_level(), QualityLevel::High); + assert_eq!(controller.last_reason(), QualityTransitionReason::None); + assert_eq!(controller.evaluate(&gpu_overload), QualityAction::Maintain); + } + + #[test] + fn test_reset_for_new_content_keeps_level_when_adaptive_mode_is_off() { + let mut controller = QualityController::new(); + controller.set_enabled(false); + controller.set_manual_level(QualityLevel::Low); + + controller.reset_for_new_content(); + + assert_eq!(controller.current_level(), QualityLevel::Low); + } } From 34315cfbeab9aaf5cc0d99e9de75b21e60d646b3 Mon Sep 17 00:00:00 2001 From: luis henrique Date: Sat, 21 Nov 2026 00:46:03 -0300 Subject: [PATCH 3/4] fix(bridge): pass content frame rate to the quality controller for network sources Only local and HTTP playback told the quality controller the video's frame rate, so refresh-rate cadence matching never ran for SMB, FTP, SFTP, NFS or WebDAV files. All six playback entry points now set it after a successful load and clear it on failure, through one helper. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DKhrqTrv7b11FgaPF1e7FV --- rust/bridge/src/lib.rs | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/rust/bridge/src/lib.rs b/rust/bridge/src/lib.rs index c1090d5..56c7bd6 100644 --- a/rust/bridge/src/lib.rs +++ b/rust/bridge/src/lib.rs @@ -506,6 +506,14 @@ pub extern "C" fn evaluate_upscaling_ffi( static QUALITY_CONTROLLER: Lazy> = Lazy::new(|| Mutex::new(media_logic::quality::QualityController::new())); +/// Lets the governor pick a display refresh rate that is a whole multiple of the video's +/// frame rate; 0.0 means unknown. +fn set_quality_content_fps(fps: f32) { + if let Ok(mut qc) = QUALITY_CONTROLLER.lock() { + qc.set_content_fps(fps); + } +} + fn reset_quality_for_new_video() { if let Ok(mut qc) = QUALITY_CONTROLLER.lock() { qc.reset_for_new_content(); @@ -1329,7 +1337,6 @@ pub extern "C" fn start_smb_playback( } reset_3d_mode(); - reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { @@ -1340,8 +1347,10 @@ pub extern "C" fn start_smb_playback( log(6, &format!("Error loading SMB video: {:?}", e)); } set_last_playback_error(format!("{:?}", e)); + set_quality_content_fps(0.0); } else { apply_screen_mode_after_load(&controller); + set_quality_content_fps(controller.video_fps()); unsafe { log(4, "SMB video loaded successfully!"); } @@ -1543,7 +1552,6 @@ pub extern "C" fn start_ftp_playback( } reset_3d_mode(); - reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { @@ -1554,8 +1562,10 @@ pub extern "C" fn start_ftp_playback( log(6, &format!("Error loading FTP video: {:?}", e)); } set_last_playback_error(format!("{:?}", e)); + set_quality_content_fps(0.0); } else { apply_screen_mode_after_load(&controller); + set_quality_content_fps(controller.video_fps()); unsafe { log(4, "FTP video loaded successfully!"); } @@ -1711,7 +1721,6 @@ pub extern "C" fn start_sftp_playback( } reset_3d_mode(); - reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { @@ -1722,8 +1731,10 @@ pub extern "C" fn start_sftp_playback( log(6, &format!("Error loading SFTP video: {:?}", e)); } set_last_playback_error(format!("{:?}", e)); + set_quality_content_fps(0.0); } else { apply_screen_mode_after_load(&controller); + set_quality_content_fps(controller.video_fps()); unsafe { log(4, "SFTP video loaded successfully!"); } @@ -1932,7 +1943,6 @@ pub extern "C" fn start_nfs_playback( } reset_3d_mode(); - reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { @@ -1943,8 +1953,10 @@ pub extern "C" fn start_nfs_playback( log(6, &format!("Error loading NFS video: {:?}", e)); } set_last_playback_error(format!("{:?}", e)); + set_quality_content_fps(0.0); } else { apply_screen_mode_after_load(&controller); + set_quality_content_fps(controller.video_fps()); unsafe { log(4, "NFS video loaded successfully!"); } @@ -2112,7 +2124,6 @@ pub extern "C" fn start_webdav_playback( } reset_3d_mode(); - reset_quality_for_new_video(); DOWNLOAD_MANAGER.set_playback_active(true); if let Ok(mut controller) = CONTROLLER.lock() { @@ -2123,8 +2134,10 @@ pub extern "C" fn start_webdav_playback( log(6, &format!("Error loading WebDAV video: {:?}", e)); } set_last_playback_error(format!("{:?}", e)); + set_quality_content_fps(0.0); } else { apply_screen_mode_after_load(&controller); + set_quality_content_fps(controller.video_fps()); unsafe { log(4, "WebDAV video loaded successfully!"); } @@ -2446,7 +2459,6 @@ pub extern "C" fn start_video_playback(path: *const std::os::raw::c_char, start_ } reset_3d_mode(); - reset_quality_for_new_video(); if path_str.starts_with("http://") || path_str.starts_with("https://") { DOWNLOAD_MANAGER.set_playback_active(true); @@ -2459,14 +2471,10 @@ pub extern "C" fn start_video_playback(path: *const std::os::raw::c_char, start_ log(6, &format!("Error loading video: {:?}", e)); } set_last_playback_error(format!("{:?}", e)); - if let Ok(mut qc) = QUALITY_CONTROLLER.lock() { - qc.set_content_fps(0.0); - } + set_quality_content_fps(0.0); } else { apply_screen_mode_after_load(&controller); - if let Ok(mut qc) = QUALITY_CONTROLLER.lock() { - qc.set_content_fps(controller.video_fps()); - } + set_quality_content_fps(controller.video_fps()); unsafe { log(4, "Video loaded successfully!"); } From 3e6f4563b2783426db20622e47294c906a5ae805 Mon Sep 17 00:00:00 2001 From: luis henrique Date: Sat, 21 Nov 2026 01:55:55 -0300 Subject: [PATCH 4/4] fix(core): pass the frame rate to the hardware video decoder The decoder was configured with realtime priority but without KEY_FRAME_RATE or KEY_OPERATING_RATE, which MediaFormat documents as the rate a realtime codec plans its resources for. Both are now set from the stream's average frame rate when the container declares it. On Quest 3 with an 8192x4096 HEVC Main 10 file at ~122 Mbps, one ~2-minute run each: late-dropped frames went from 3.85 to 2.40 per second on average (p95 10.9 -> 8.0) and decoded fps from 56.1 to 57.1. The stream still does not hold a steady 60 fps. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DKhrqTrv7b11FgaPF1e7FV --- rust/core/src/playback.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rust/core/src/playback.rs b/rust/core/src/playback.rs index c9f0264..bf057a6 100644 --- a/rust/core/src/playback.rs +++ b/rust/core/src/playback.rs @@ -647,6 +647,13 @@ impl PlaybackController { format.set_i32("max-input-size", max_input_size); // Prioridade de decodificação realtime (0 = realtime / baixa latência) format.set_i32("priority", 0); + // With realtime priority the codec sizes its clocks and bandwidth from the frame rate + // (MediaFormat KEY_FRAME_RATE / KEY_OPERATING_RATE). Without it, an 8192x4096 HEVC + // stream at ~122 Mbps decoded at ~47-57 of 60 fps on Quest 3. + if video_fps > 0.0 { + format.set_f32("frame-rate", video_fps); + format.set_f32("operating-rate", video_fps); + } // Repassa ao MediaCodec o espaco de cor declarado pelo container: o // dataspace do buffer decodificado vira o `suggestedYcbcrModel/Range`