From a377d219fc084b247da7ebf0dfef202b101bd7c0 Mon Sep 17 00:00:00 2001 From: Sapphire Date: Wed, 24 Dec 2025 01:46:36 -0600 Subject: [PATCH 1/4] input: move hand tracker into device --- src/input.rs | 43 ++++++++++++++++++------------------ src/input/action_manifest.rs | 24 ++------------------ src/input/devices.rs | 18 +++++++++++++-- src/input/skeletal.rs | 5 +---- src/openxr_data.rs | 9 ++++++++ 5 files changed, 49 insertions(+), 50 deletions(-) diff --git a/src/input.rs b/src/input.rs index c5a71fc0..6079100e 100644 --- a/src/input.rs +++ b/src/input.rs @@ -275,10 +275,7 @@ enum ActionData { last_value: (AtomicF32, AtomicF32), }, Pose, - Skeleton { - hand: Hand, - hand_tracker: Option, - }, + Skeleton(Hand), Haptic(xr::Action), } @@ -559,22 +556,11 @@ impl vr::IVRInput010_Interface for Input { }; get_action_from_handle!(self, handle, session_data, action); - let ActionData::Skeleton { hand, hand_tracker } = action else { + let ActionData::Skeleton(hand) = action else { return vr::EVRInputError::WrongType; }; - if let Some(hand_tracker) = hand_tracker.as_ref() { - self.get_bones_from_hand_tracking( - &session_data, - transform_space, - hand_tracker, - *hand, - transforms, - ) - } else { - self.get_estimated_bones(&session_data, transform_space, *hand, transforms); - } - + self.get_bones_from_hand_tracking(&session_data, transform_space, *hand, transforms); vr::EVRInputError::None } fn GetSkeletalTrackingLevel( @@ -583,7 +569,7 @@ impl vr::IVRInput010_Interface for Input { level: *mut vr::EVRSkeletalTrackingLevel, ) -> vr::EVRInputError { get_action_from_handle!(self, action, data, action); - let ActionData::Skeleton { hand, .. } = action else { + let ActionData::Skeleton(hand) = action else { return vr::EVRInputError::WrongType; }; @@ -627,7 +613,7 @@ impl vr::IVRInput010_Interface for Input { }; get_action_from_handle!(self, handle, session_data, action); - let ActionData::Skeleton { hand, .. } = action else { + let ActionData::Skeleton(hand) = action else { return vr::EVRInputError::WrongType; }; @@ -688,7 +674,7 @@ impl vr::IVRInput010_Interface for Input { return vr::EVRInputError::InvalidHandle; }; let origin = match loaded.try_get_action(action) { - Ok(ActionData::Skeleton { hand, .. }) => match hand { + Ok(ActionData::Skeleton(hand)) => match hand { Hand::Left => self.left_hand_key.data().as_ffi(), Hand::Right => self.right_hand_key.data().as_ffi(), }, @@ -823,7 +809,7 @@ impl vr::IVRInput010_Interface for Input { } } } - Ok(ActionData::Skeleton { hand, .. }) => { + Ok(ActionData::Skeleton(hand)) => { if subaction_path != xr::Path::NULL { return vr::EVRInputError::InvalidDevice; } @@ -1372,8 +1358,21 @@ impl Input { if let Some(controller) = controller.as_mut() { controller.interaction_profile = Some(p); } else { + let hand_tracker = session_data + .session + .create_hand_tracker(hand.into()) + .inspect_err(|e| { + if !matches!( + *e, + xr::sys::Result::ERROR_EXTENSION_NOT_PRESENT + | xr::sys::Result::ERROR_FEATURE_UNSUPPORTED + ) { + log::warn!("Failed to create hand tracker for hand {hand:?}: {e}"); + } + }) + .ok(); devices_to_create.push(( - TrackedDeviceType::Controller { hand }, + TrackedDeviceType::Controller { hand, hand_tracker }, Some(profile_path), Some(p), )); diff --git a/src/input/action_manifest.rs b/src/input/action_manifest.rs index 8ed9894d..68c31878 100644 --- a/src/input/action_manifest.rs +++ b/src/input/action_manifest.rs @@ -104,7 +104,6 @@ impl Input { let actions = load_actions( &self.openxr.instance, - &session_data.session, english.as_ref(), &mut sets, manifest.actions, @@ -441,7 +440,6 @@ fn create_action( type LoadedActionDataMap = HashMap; fn load_actions( instance: &xr::Instance, - session: &xr::Session, english: Option<&Localization>, sets: &mut HashMap, actions: Vec, @@ -485,25 +483,7 @@ fn load_actions( ActionType::Pose(data) => (&data.name, Pose), ActionType::Skeleton(SkeletonData { skeleton, data }) => { trace!("Creating skeleton action {}", data.name.path); - let hand_tracker = match session.create_hand_tracker(match skeleton { - Hand::Left => xr::Hand::LEFT, - Hand::Right => xr::Hand::RIGHT, - }) { - Ok(t) => Some(t), - Err( - xr::sys::Result::ERROR_EXTENSION_NOT_PRESENT - | xr::sys::Result::ERROR_FEATURE_UNSUPPORTED, - ) => None, - Err(other) => panic!("Creating hand tracker failed: {other:?}"), - }; - - ( - &data.name, - Skeleton { - hand: *skeleton, - hand_tracker, - }, - ) + (&data.name, Skeleton(*skeleton)) } ActionType::Vibration(data) => (&data.name, Haptic(create_action!(xr::Haptic, data))), }; @@ -1503,7 +1483,7 @@ fn handle_skeleton_bindings( }; match &context.actions[&output.path] { - super::ActionData::Skeleton { hand, .. } => { + super::ActionData::Skeleton(hand) => { let bound_hand = match path.as_str() { "/user/hand/left/input/skeleton/left" => Hand::Left, "/user/hand/right/input/skeleton/right" => Hand::Right, diff --git a/src/input/devices.rs b/src/input/devices.rs index 6ad8c007..d7d8ffb6 100644 --- a/src/input/devices.rs +++ b/src/input/devices.rs @@ -19,6 +19,7 @@ pub enum TrackedDeviceType { Hmd, Controller { hand: Hand, + hand_tracker: Option, }, #[cfg(feature = "monado")] GenericTracker { @@ -31,7 +32,7 @@ impl std::fmt::Debug for TrackedDeviceType { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { TrackedDeviceType::Hmd => write!(f, "HMD"), - TrackedDeviceType::Controller { hand } => write!(f, "Controller ({:?})", hand), + TrackedDeviceType::Controller { hand, .. } => write!(f, "Controller ({:?})", hand), #[cfg(feature = "monado")] TrackedDeviceType::GenericTracker { serial, .. } => { write!(f, "Generic Tracker ({})", serial.to_string_lossy()) @@ -158,6 +159,19 @@ impl TrackedDevice { *pose_cache } + pub fn get_hand_skeleton( + &self, + xr_data: &OpenXrData, + base: &xr::Space, + ) -> Option { + let TrackedDeviceType::Controller { hand_tracker, .. } = self.get_type() else { + return None; + }; + + base.locate_hand_joints(hand_tracker.as_ref()?, xr_data.display_time.get()) + .unwrap_or_default() + } + pub fn clear_pose_cache(&self) { std::mem::take(&mut *self.pose_cache.lock().unwrap()); } @@ -184,7 +198,7 @@ impl TrackedDevice { fn get_string_property(&self, property: vr::ETrackedDeviceProperty) -> Option<&CStr> { let hand = match self.device_type { - TrackedDeviceType::Controller { hand } => hand, + TrackedDeviceType::Controller { hand, .. } => hand, _ => Hand::Left, }; diff --git a/src/input/skeletal.rs b/src/input/skeletal.rs index d74e1d5f..9e9f79f7 100644 --- a/src/input/skeletal.rs +++ b/src/input/skeletal.rs @@ -14,19 +14,16 @@ use std::f32::consts::{FRAC_PI_2, PI}; use std::time::Instant; impl Input { - /// Returns false if hand tracking data couldn't be generated for some reason. pub(super) fn get_bones_from_hand_tracking( &self, session_data: &SessionData, space: vr::EVRSkeletalTransformSpace, - hand_tracker: &xr::HandTracker, hand: Hand, transforms: &mut [vr::VRBoneTransform_t], ) { use HandSkeletonBone::*; let pose_data = session_data.input_data.pose_data.get().unwrap(); - let display_time = self.openxr.display_time.get(); let devices = session_data.input_data.devices.read().unwrap(); let Some(controller) = devices.get_controller(hand) else { @@ -43,7 +40,7 @@ impl Input { return; }; - let Some(joints) = raw.locate_hand_joints(hand_tracker, display_time).unwrap() else { + let Some(joints) = controller.get_hand_skeleton(&self.openxr, &raw) else { self.get_estimated_bones(session_data, space, hand, transforms); return; }; diff --git a/src/openxr_data.rs b/src/openxr_data.rs index 333673de..ab498044 100644 --- a/src/openxr_data.rs +++ b/src/openxr_data.rs @@ -687,6 +687,15 @@ impl From for vr::ETrackedControllerRole { } } +impl From for xr::HandEXT { + fn from(hand: Hand) -> Self { + match hand { + Hand::Left => xr::HandEXT::LEFT, + Hand::Right => xr::HandEXT::RIGHT, + } + } +} + /// Taken from: https://github.com/bitshifter/glam-rs/issues/536 /// Decompose the rotation on to 2 parts. /// From 6f0763e2f8b5cd81c9bd8530d6493b5f4affa536 Mon Sep 17 00:00:00 2001 From: Sapphire Date: Tue, 17 Feb 2026 02:53:50 -0600 Subject: [PATCH 2/4] input: cache openxr hand skeleton --- src/input.rs | 6 +++++- src/input/devices.rs | 23 ++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/input.rs b/src/input.rs index 6079100e..d6aac408 100644 --- a/src/input.rs +++ b/src/input.rs @@ -1372,7 +1372,11 @@ impl Input { }) .ok(); devices_to_create.push(( - TrackedDeviceType::Controller { hand, hand_tracker }, + TrackedDeviceType::Controller { + hand, + hand_tracker, + skeleton_cache: Mutex::new(Default::default()), + }, Some(profile_path), Some(p), )); diff --git a/src/input/devices.rs b/src/input/devices.rs index d7d8ffb6..dc8841c7 100644 --- a/src/input/devices.rs +++ b/src/input/devices.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::sync::Mutex; @@ -20,6 +21,7 @@ pub enum TrackedDeviceType { Controller { hand: Hand, hand_tracker: Option, + skeleton_cache: Mutex>>, }, #[cfg(feature = "monado")] GenericTracker { @@ -164,16 +166,31 @@ impl TrackedDevice { xr_data: &OpenXrData, base: &xr::Space, ) -> Option { - let TrackedDeviceType::Controller { hand_tracker, .. } = self.get_type() else { + let TrackedDeviceType::Controller { + hand_tracker, + skeleton_cache, + .. + } = self.get_type() + else { return None; }; + let mut skeleton_cache = skeleton_cache.lock().unwrap(); + if let Some(skeleton) = skeleton_cache.get(&base.as_raw().into_raw()) { + return *skeleton; + } - base.locate_hand_joints(hand_tracker.as_ref()?, xr_data.display_time.get()) - .unwrap_or_default() + let joints = base + .locate_hand_joints(hand_tracker.as_ref()?, xr_data.display_time.get()) + .unwrap_or_default(); + skeleton_cache.insert(base.as_raw().into_raw(), joints); + joints } pub fn clear_pose_cache(&self) { std::mem::take(&mut *self.pose_cache.lock().unwrap()); + if let TrackedDeviceType::Controller { skeleton_cache, .. } = self.get_type() { + skeleton_cache.lock().unwrap().clear(); + } } pub fn has_connected_changed(&mut self) -> bool { From 4e33b2cc4facdc8e663749f93ab14ff832a5fb6c Mon Sep 17 00:00:00 2001 From: Sarah Date: Thu, 20 Mar 2025 00:44:36 +0100 Subject: [PATCH 3/4] input: implement skeletal summary This is a direct port of OC's implementation, which isn't perfect. The calculated curl is not correct for the thumb, and on Quest the curl for the other fingers seems to max out at 0.8, but it's good enough to get gestures working in VRChat on Knuckles. Co-authored-by: Sapphire --- src/input.rs | 22 +++++--- src/input/skeletal.rs | 127 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/src/input.rs b/src/input.rs index d6aac408..99d1bd43 100644 --- a/src/input.rs +++ b/src/input.rs @@ -527,17 +527,21 @@ impl vr::IVRInput010_Interface for Input { fn GetSkeletalSummaryData( &self, action: vr::VRActionHandle_t, - _: vr::EVRSummaryType, + summary_type: vr::EVRSummaryType, data: *mut vr::VRSkeletalSummaryData_t, ) -> vr::EVRInputError { - crate::warn_unimplemented!("GetSkeletalSummaryData"); - get_action_from_handle!(self, action, session_data, _action); - unsafe { - data.write(vr::VRSkeletalSummaryData_t { - flFingerSplay: [0.2; 4], - flFingerCurl: [0.0; 5], - }) - } + get_action_from_handle!(self, action, session_data, action); + + let ActionData::Skeleton(hand) = action else { + return vr::EVRInputError::WrongType; + }; + + let Some(data) = (unsafe { data.as_mut() }) else { + return vr::EVRInputError::InvalidParam; + }; + + self.get_bone_summary_from_hand_tracking(&session_data, summary_type, data, *hand); + vr::EVRInputError::None } fn GetSkeletalBoneData( diff --git a/src/input/skeletal.rs b/src/input/skeletal.rs index 9e9f79f7..3cbef531 100644 --- a/src/input/skeletal.rs +++ b/src/input/skeletal.rs @@ -162,6 +162,112 @@ impl Input { *self.skeletal_tracking_level.write().unwrap() = vr::EVRSkeletalTrackingLevel::Full; } + pub(super) fn get_bone_summary_from_hand_tracking( + &self, + session_data: &SessionData, + summary_type: vr::EVRSummaryType, + summary_data: &mut vr::VRSkeletalSummaryData_t, + hand: Hand, + ) { + let pose_data = session_data.input_data.pose_data.get().unwrap(); + let devices = session_data.input_data.devices.read().unwrap(); + + let Some(controller) = devices.get_controller(hand) else { + self.get_estimated_bone_summary(session_data, summary_type, summary_data, hand); + return; + }; + + let Some(raw) = match hand { + Hand::Left => &pose_data.left_space, + Hand::Right => &pose_data.right_space, + } + .try_get_or_init_raw(&controller.interaction_profile, session_data, pose_data) else { + self.get_estimated_bone_summary(session_data, summary_type, summary_data, hand); + return; + }; + + let Some(joints) = controller.get_hand_skeleton(&self.openxr, &raw) else { + self.get_estimated_bone_summary(session_data, summary_type, summary_data, hand); + return; + }; + + let joints: Box<[_]> = joints + .into_iter() + .map(|joint_location| { + let position = joint_location.pose.position; + let orientation = joint_location.pose.orientation; + ( + Vec3::from_array([position.x, position.y, position.z]), + Quat::from_xyzw(orientation.x, orientation.y, orientation.z, orientation.w), + ) + }) + .collect(); + + // FIXME: calculate splay + summary_data.flFingerSplay.fill(0.2); + + for (i, out_curl) in summary_data.flFingerCurl.iter_mut().enumerate() { + let (metacarpal, proximal, tip) = match i { + 0 => ( + joints[xr::HandJoint::THUMB_METACARPAL], + joints[xr::HandJoint::THUMB_PROXIMAL], + joints[xr::HandJoint::THUMB_TIP], + ), + 1 => ( + joints[xr::HandJoint::INDEX_METACARPAL], + joints[xr::HandJoint::INDEX_PROXIMAL], + joints[xr::HandJoint::INDEX_TIP], + ), + 2 => ( + joints[xr::HandJoint::MIDDLE_METACARPAL], + joints[xr::HandJoint::MIDDLE_PROXIMAL], + joints[xr::HandJoint::MIDDLE_TIP], + ), + 3 => ( + joints[xr::HandJoint::RING_METACARPAL], + joints[xr::HandJoint::RING_PROXIMAL], + joints[xr::HandJoint::RING_TIP], + ), + 4 => ( + joints[xr::HandJoint::LITTLE_METACARPAL], + joints[xr::HandJoint::LITTLE_PROXIMAL], + joints[xr::HandJoint::LITTLE_TIP], + ), + _ => unreachable!(), + }; + + // Vector pointing from the knuckle to the metacarpal + let proximal_metacarpal_delta = metacarpal.0 - proximal.0; + // Vector pointing from the knuckle to the tip of the finger + let tip_proximal_delta = tip.0 - proximal.0; + + // The dotproduct will give us the angle between the two vectors + let dot = proximal_metacarpal_delta.dot(tip_proximal_delta); + let a = proximal_metacarpal_delta.length(); + let b = tip_proximal_delta.length(); + + let curl = if a == 0.0 || b == 0.0 { + // If the joints converge, say the finger is fully curled + 1.0 + } else { + // Isolate cos(angle) + let ang_cos = (dot / (a * b)).clamp(-1.0, 1.0); + // Convert the angle to radians + let ang = ang_cos.acos(); + + // When the finger is straight, the angle is 180deg (PI radians) + // When the finger is fully curled, the angle is (theoretically) 0 + 1.0 - (ang / PI) + }; + + // But in the real world, when a finger is curled, an angle of 0 is physically + // not possible. The curl maxes out at around 0.8, give or take some depending on + // the user's hands. Remap the value so >=0.8 (arbitrary value) means fully curled + const MAX_CURL: f32 = 0.8; + *out_curl = (curl / MAX_CURL).clamp(0.0, 1.0); + } + } + pub(super) fn get_estimated_bones( &self, session_data: &SessionData, @@ -209,6 +315,27 @@ impl Input { *self.skeletal_tracking_level.write().unwrap() = vr::EVRSkeletalTrackingLevel::Estimated; } + pub(super) fn get_estimated_bone_summary( + &self, + session_data: &SessionData, + _: vr::EVRSummaryType, + summary_data: &mut vr::VRSkeletalSummaryData_t, + hand: Hand, + ) { + let state = self.get_finger_state(session_data, hand); + + *summary_data = vr::VRSkeletalSummaryData_t { + flFingerSplay: [0.2; 4], + flFingerCurl: [ + state.thumb, + state.index, + state.middle, + state.ring, + state.pinky, + ], + }; + } + fn get_finger_state(&self, session_data: &SessionData, hand: Hand) -> FingerState { // Determines the speed at which fingers follow the input states // This value seems to feel right for both analog inputs and binary ones (like vive wands) From ec556416d6dc504452fba0f8aadd72d4d193a60f Mon Sep 17 00:00:00 2001 From: Sarah Date: Tue, 17 Feb 2026 03:16:04 -0600 Subject: [PATCH 4/4] input/skeletal: use estimated curl for the thumb on Knuckles controllers --- src/input/profiles/knuckles.rs | 2 ++ src/input/skeletal.rs | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/input/profiles/knuckles.rs b/src/input/profiles/knuckles.rs index fafce4d5..9fa00830 100644 --- a/src/input/profiles/knuckles.rs +++ b/src/input/profiles/knuckles.rs @@ -144,6 +144,8 @@ impl InteractionProfile for Knuckles { .leftright("input/thumbstick/touch") .into_iter() .chain(stp.leftright("input/trackpad/touch")) + .chain(stp.leftright("input/a/touch")) + .chain(stp.leftright("input/b/touch")) .collect(), index_touch: stp.leftright("input/trigger/touch"), index_curl: stp.leftright("input/trigger/value"), diff --git a/src/input/skeletal.rs b/src/input/skeletal.rs index 3cbef531..ffc1995e 100644 --- a/src/input/skeletal.rs +++ b/src/input/skeletal.rs @@ -207,6 +207,21 @@ impl Input { summary_data.flFingerSplay.fill(0.2); for (i, out_curl) in summary_data.flFingerCurl.iter_mut().enumerate() { + // For Knuckles, the skeleton thumb tries to accurately match where the physical + // thumb is, e.g. the curl depends on which part of the touchpad is being touched, + // or how the thumbstick is being pushed, but in GetSkeletalSummaryData with + // EVRSummaryType::FromDevice the curl is set to 1 if any of touchpad, A/B, + // or thumbstick is being touched, so just use the skeletal input actions to + // determine the curl value for the thumb. + if i == 0 + && controller.interaction_profile.is_some_and(|p| { + p.profile_path() == "/interaction_profiles/valve/index_controller" + }) + { + *out_curl = self.get_finger_state(session_data, hand).thumb; + continue; + } + let (metacarpal, proximal, tip) = match i { 0 => ( joints[xr::HandJoint::THUMB_METACARPAL],