From 52064131b70ed192c7746b3f3aaebc8b5d320f30 Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Sat, 20 Dec 2025 20:23:36 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BB=BF=E7=9C=9F?= =?UTF-8?q?=E8=BF=90=E8=A1=8C=E7=9A=84=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config.yaml | 2 + .../simulation.xml | 456 ++++++++++++++++++ 2 files changed, 458 insertions(+) create mode 100644 src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml create mode 100644 src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulation.xml diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml new file mode 100644 index 0000000000..74b0cdcb77 --- /dev/null +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml @@ -0,0 +1,2 @@ +dt: 0.01 # 仿真步长(调慢一点,看得清) +render_mode: "human" # 开启可视化! \ No newline at end of file diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulation.xml b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulation.xml new file mode 100644 index 0000000000..8f8f98e537 --- /dev/null +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulation.xml @@ -0,0 +1,456 @@ + + + + \ No newline at end of file From 1b67b9ef36997030215d91920c0030c22ae3f059 Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Sat, 20 Dec 2025 20:34:08 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=96=87=E4=BB=B6,=E7=A7=BB=E5=8A=A8README.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{mobl_arms_index_pointing => }/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/HCL_interaction_task/{mobl_arms_index_pointing => }/README.md (98%) diff --git a/src/HCL_interaction_task/mobl_arms_index_pointing/README.md b/src/HCL_interaction_task/README.md similarity index 98% rename from src/HCL_interaction_task/mobl_arms_index_pointing/README.md rename to src/HCL_interaction_task/README.md index b0d81733fa..9841a2586d 100644 --- a/src/HCL_interaction_task/mobl_arms_index_pointing/README.md +++ b/src/HCL_interaction_task/README.md @@ -86,7 +86,7 @@ pip install -e . ### 4. 模型文件准备 完整模型文件集(包含40+手部精细模型文件)可通过以下链接下载: -[完整模型文件集网盘链接](https://pan.baidu.com/s/1SN5SWpyfKR7KYDbE8lzvBw?pwd=9e9u) +[完整模型文件集网盘链接](https://pan.baidu.com/s/1sA0BgEPRgxXTqe6ZdEm7Sg?pwd=rq8e 提取码: rq8e) 下载后解压到 `mujoco_ros_demo/config/assets/` 目录 From dfa65cd714ee8d5696a7e80ba0191b8a4820593a Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Sun, 21 Dec 2025 14:35:35 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=EF=BC=8C=E9=A9=B1=E5=8A=A8=E4=BB=BF=E7=9C=9F=E8=BF=90=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simulator.py | 223 +++++++++++------- 1 file changed, 139 insertions(+), 84 deletions(-) diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py index 5a48b643d6..12a44016ac 100644 --- a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py @@ -1,128 +1,183 @@ import mujoco -import mujoco.viewer # 强制导入viewer(放弃兼容旧版本,优先保证运行) +import mujoco.viewer import numpy as np import yaml import time import os -# 解决Linux GLX错误:优先用osmesa,若不行则用glfw(备选) -os.environ['MUJOCO_GL'] = 'osmesa' -# 增加环境变量,避免viewer闪退 -os.environ['MJPYTHON_FRAMEWORK'] = 'gtk3' +# 兼容所有MuJoCo版本+远程环境 +os.environ['MUJOCO_GL'] = 'egl' +os.environ['MJPYTHON_FRAMEWORK'] = 'none' +os.environ['MESA_GL_VERSION_OVERRIDE'] = '3.3' class IndexSimulator: def __init__(self, config_path, model_path): - # 1. 加载配置 with open(config_path, 'r') as f: self.config = yaml.safe_load(f) - # 2. 加载MuJoCo模型 self.model = mujoco.MjModel.from_xml_path(model_path) self.data = mujoco.MjData(self.model) - # 3. 检查qpos长度(关键!避免索引错误) - self.qpos_len = len(self.data.qpos) - print(f"✅ 模型加载成功!qpos长度:{self.qpos_len}") - self.move_joints = 3 if self.qpos_len >=3 else self.qpos_len - - # 4. 初始化核心变量 + # 1. 模型关键信息+关节range(稳定核心) + print("\n📌 模型核心信息:") + print(f"关节总数:{self.model.njnt} | 执行器数:{self.model.nu} | qpos长度:{len(self.data.qpos)}") + + self.key_joints = { + 15: { + "name": "shoulder_rot (肩关节旋转)", + "range": (-1.57, 0.349), + "step": 0.02, + "current_step_dir": 0.02 # 运动方向 + }, + 16: { + "name": "elbow_flexion (肘关节弯曲)", + "range": (0, 2.26893), + "step": 0.03, + "current_step_dir": 0.03 + }, + 17: { + "name": "pro_sup (前臂旋前旋后)", + "range": (-1.5708, 1.5708), + "step": 0.015, + "current_step_dir": 0.015 + } + } + + # 打印关节初始信息 + print("\n📌 关键关节(带range限制):") + for jnt_id, info in self.key_joints.items(): + if jnt_id < len(self.data.qpos): + init_val = (info['range'][0] + info['range'][1]) / 2 + print(f"关节{jnt_id}:{info['name']} | range:{info['range']} | 初始值:{init_val:.2f} rad") + + # 2. 仿真状态(极简,避免段错误) self.is_running = True self.viewer = None - self.button_names = ["button-0", "button-1", "button-2", "button-3"] - self.button_touched = {name: False for name in self.button_names} + self.print_interval = 50 + self.current_step = 0 + self.button_touched = {"button-0":False, "button-1":False, "button-2":False, "button-3":False} self.finger_geom_name = "hand_2distph" def reset(self): - """重置仿真:安全赋值qpos""" + """重置关节到range中间值""" mujoco.mj_resetData(self.model, self.data) - # 安全赋值:只给存在的关节赋值 - if self.qpos_len >= 1: - self.data.qpos[0] = 0.0 # X方向 - if self.qpos_len >= 2: - self.data.qpos[1] = 0.0 # Y方向 - if self.qpos_len >= 3: - self.data.qpos[2] = 0.0 # Z方向 - self.button_touched = {name: False for name in self.button_names} + for jnt_id, info in self.key_joints.items(): + if jnt_id < len(self.data.qpos): + init_val = (info['range'][0] + info['range'][1]) / 2 + self.data.qpos[jnt_id] = init_val + self.key_joints[jnt_id]['current_step_dir'] = info['step'] # 重置方向 + self.current_step = 0 self.is_running = True return self.data.qpos.copy() - def step(self, action=None): - """每一步仿真:安全移动手指""" - # 1. 默认动作:向按钮移动(放慢速度,避免瞬移) - if action is None: - action = np.array([0.001, 0.0, 0.0]) # 把步长从0.01改成0.001,移动更慢更稳 - - # 2. 安全更新qpos(只更新存在的关节) - for i in range(min(len(action), self.move_joints)): - self.data.qpos[i] += action[i] - - # 3. 推进仿真 + def step(self): + """单步仿真:稳定+慢速,无错误""" + if not self.is_running: + return self.data.qpos.copy() + + # 1. 关节慢速摆动(不超range) + for jnt_id, info in self.key_joints.items(): + if jnt_id >= len(self.data.qpos): + continue + + current_val = self.data.qpos[jnt_id] + new_val = current_val + info['current_step_dir'] + + # 到边界反向 + if new_val > info['range'][1]: + self.key_joints[jnt_id]['current_step_dir'] = -info['step'] + new_val = info['range'][1] - 0.01 + elif new_val < info['range'][0]: + self.key_joints[jnt_id]['current_step_dir'] = info['step'] + new_val = info['range'][0] + 0.01 + + # 安全赋值 + self.data.qpos[jnt_id] = np.clip(new_val, info['range'][0], info['range'][1]) + + # 2. 推进仿真 mujoco.mj_step(self.model, self.data) - # 4. 检测碰撞 - self._check_button_collision() + # 3. 定期打印(减少刷屏) + if self.current_step % self.print_interval == 0: + print(f"\n📌 step{self.current_step} 关节角度(直观度数):") + for jnt_id, info in self.key_joints.items(): + if jnt_id < len(self.data.qpos): + rad = self.data.qpos[jnt_id] + deg = rad * 57.3 # 弧度转角度 + print(f" {info['name']}:{rad:.2f} rad ≈ {deg:.0f}°") + + # 4. 渲染(容错处理,避免段错误) + if self.viewer: + try: + self.viewer.sync() + time.sleep(0.03) # 慢速,肉眼看清 + except Exception as e: + print(f"⚠️ 渲染同步警告(不影响仿真):{e}") + pass + self.current_step += 1 return self.data.qpos.copy() def run_simulation(self): - """ - 核心修改:主动创建Viewer+强制循环,直到手动关闭 - """ - print("✅ 仿真器初始化成功!") - print("👉 可视化窗口已弹出,手指正在向按钮移动...(按Ctrl+C终止)") + """核心:仅Ctrl+C中断,无窗口检测(解决段错误)""" + print("\n✅ 仿真启动成功!") + print("👉 手臂会慢速稳定摆动,按【Ctrl+C】终止仿真(关闭窗口需手动Ctrl+C)") + print("👉 关节严格限制在安全范围,无NaN/段错误!") self.reset() - # 主动创建Viewer实例(放弃passive模式,改用主动模式) + # 启动可视化(最大容错) try: - with mujoco.viewer.launch(self.model, self.data) as self.viewer: - # 调整视角 - self.viewer.cam.azimuth = 135 - self.viewer.cam.elevation = -15 - self.viewer.cam.distance = 1.2 - self.viewer.cam.lookat = [0.4, 0.0, 0.4] - - # 强制循环:直到手动关闭窗口/按Ctrl+C - while self.is_running and self.viewer.is_running(): - # 执行仿真步 - self.step() - # 控制帧率(每秒50帧,不会卡) - time.sleep(0.02) - # 同步viewer(主动模式必须手动sync) - self.viewer.sync() - - except KeyboardInterrupt: - # 按Ctrl+C优雅退出 - print("\n⚠️ 检测到Ctrl+C,正在退出仿真...") + self.viewer = mujoco.viewer.launch_passive(self.model, self.data) + # 优化视角 + self.viewer.cam.azimuth = 135 + self.viewer.cam.elevation = -15 + self.viewer.cam.distance = 0.6 + self.viewer.cam.lookat = [0.45, -0.15, 0.8] + print("✅ 可视化窗口启动成功!能看到手臂慢速摆动") except Exception as e: - print(f"⚠️ 可视化启动失败,改用无窗口模式运行:{e}") - # 无窗口模式:循环100秒后退出 - start_time = time.time() - while time.time() - start_time < 100: + print(f"⚠️ 可视化启动失败(不影响动作):{e}") + print("📢 无窗口模式:终端打印关节角度,确认动作真实发生!") + + # ========== 终极循环:仅Ctrl+C中断,无窗口检测 ========== + try: + while self.is_running: self.step() - time.sleep(0.02) + except KeyboardInterrupt: + print("\n\n⚠️ 检测到Ctrl+C,正在优雅退出仿真...") + self.is_running = False + # 清理资源 self.close() - - def _check_button_collision(self): - """检测碰撞(简化版)""" - for i in range(self.data.ncon): - contact = self.data.contact[i] - geom1 = self.model.geom(contact.geom1).name - geom2 = self.model.geom(contact.geom2).name - - for btn_name in self.button_names: - if (btn_name in [geom1, geom2]) and (self.finger_geom_name in [geom1, geom2]): - if not self.button_touched[btn_name]: - color_map = {"button-0":"红", "button-1":"绿", "button-2":"蓝", "button-3":"黄"} - print(f"🎉 碰到【{color_map[btn_name]}按钮】!(继续运行,不会停止)") - self.button_touched[btn_name] = True + print(f"\n✅ 仿真正常结束!共运行{self.current_step}步") + print("📊 最终关节角度:") + for jnt_id, info in self.key_joints.items(): + if jnt_id < len(self.data.qpos): + rad = self.data.qpos[jnt_id] + deg = rad * 57.3 + print(f" {info['name']}:{rad:.2f} rad ≈ {deg:.0f}°") def close(self): - """关闭资源""" + """安全关闭资源,避免段错误""" self.is_running = False if self.viewer: try: - self.viewer.close() + # 不同版本MuJoCo的viewer关闭方式兼容 + if hasattr(self.viewer, 'close'): + self.viewer.close() + else: + mujoco.viewer.close(self.viewer) except: pass - print("\n👋 仿真结束~") \ No newline at end of file + +if __name__ == "__main__": + # 替换为你的配置和模型路径 + CONFIG_PATH = "config.yaml" + MODEL_PATH = "simulation.xml" + + # 启动仿真(包裹try-except,避免核心转储) + try: + sim = IndexSimulator(CONFIG_PATH, MODEL_PATH) + sim.run_simulation() + except Exception as e: + print(f"\n❌ 仿真启动异常:{e}") + print("💡 请检查模型文件路径/配置文件是否正确") \ No newline at end of file From bd4401c2e1900398f3300793b0e5a5b7e56ad792 Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Sun, 21 Dec 2025 16:50:00 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E6=9B=B4=E6=96=B0readme.md=EF=BC=8C?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=96=B0=E5=8A=9F=E8=83=BD=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HCL_interaction_task/README.md | 141 ++++++++++++++++++----------- 1 file changed, 87 insertions(+), 54 deletions(-) diff --git a/src/HCL_interaction_task/README.md b/src/HCL_interaction_task/README.md index 9841a2586d..806c611cfb 100644 --- a/src/HCL_interaction_task/README.md +++ b/src/HCL_interaction_task/README.md @@ -2,105 +2,138 @@ ## 项目概述 -本项目融合了基于MuJoCo的人体上肢物理仿真与ROS 2机器人操作系统,实现了兼具高精度食指追踪功能与分布式通信能力的综合仿真系统。项目核心包含两大部分:一是通过MediaPipe手部追踪驱动的食指指向仿真,二是基于ROS 2的关节数据发布与感知处理模块,可实现虚拟模型与外部系统的实时数据交互。 +本项目融合了基于MuJoCo的人体上肢物理仿真与ROS 2机器人操作系统,实现了兼具高精度手势交互与分布式通信能力的综合仿真系统。项目核心包含两大功能模块: +1. **食指指向追踪**(`mobl_arms_index_pointing`):通过MediaPipe手部追踪驱动食指指向指定目标(如小球)的仿真功能 +- **功能演示**: +![ezgif-72b271013158aed5](https://github.com/user-attachments/assets/937c5994-4cfe-42ae-8f77-2eb0ebb1b41a) + +2. **面板选择反应**(`mobl_arms_index_choice_reaction`):新增的多目标面板交互模块,支持食指对虚拟面板进行选择与反应测试 +- **功能演示**: +![ezgif-283f8d513dd75e78](https://github.com/user-attachments/assets/6edd413e-8566-4b36-a8c0-fd6650ceee10) -### 演示视频 -- **食指指向功能演示**: -![ezgif-72b271013158aed5](https://github.com/user-attachments/assets/937c5994-4cfe-42ae-8f77-2eb0ebb1b41a) ## 主要特点 +### 通用特点 - **高精度骨骼模型**:包含上肢完整骨骼结构,细化手部掌骨、指骨(近节/中节/远节)的几何与物理参数 -- **增强型肌肉驱动**:新增食指相关肌肉群(FDPI、EDM等)的肌腱路径定义,提升指向动作真实性 +- **增强型肌肉驱动**:基于FDPI、EDM等食指相关肌肉群的肌腱路径定义,提升动作真实性 - **实时手势映射**:集成MediaPipe手部追踪,将真实手势实时映射到虚拟模型 -- **目标追踪功能**:支持配置目标坐标,实现食指自动指向指定位置 - **ROS 2分布式通信**:通过ROS 2节点实现关节数据发布、感知数据处理与外部系统交互 -- **模块化设计**:保持与[User-in-the-Box](https://github.com/User-in-the-Box/user-in-the-box)原项目的兼容性,同时扩展ROS 2接口 +- **模块化设计**:保持与[User-in-the-Box](https://github.com/User-in-the-Box/user-in-the-box)原项目的兼容性 -## 文件说明 +### 食指指向模块(`mobl_arms_index_pointing`) +- **目标追踪功能**:支持配置单一目标坐标(如小球),实现食指自动指向指定位置 +- **动态轨迹规划**:根据目标位置实时计算手指运动轨迹,确保平滑自然的指向动作 -| 文件名 | 描述 | -|--------|------| -| **config.yaml** | 仿真参数配置文件,包含仿真步长(dt)、渲染模式、窗口分辨率及目标位置(target_pos)等 | -| **simulation.xml** | MuJoCo模型定义文件,包含骨骼结构、肌肉肌腱参数、标记点及关节活动范围等核心信息 | -| **evaluator.py** | 程序入口脚本,通过命令行参数接收配置文件和模型文件路径,初始化并启动仿真 | -| **simulator.py** | 仿真器核心逻辑,包含MuJoCo环境初始化、Viewer适配、仿真循环控制及手势映射功能 | -| **assets/** | 模型资源文件夹,存放网格文件(.stl)和纹理文件,定义骨骼与手部的几何形状 | -| **mujoco_ros_demo/** | ROS 2功能包目录 | +### 面板选择反应模块(`mobl_arms_index_choice_reaction`) +- **多目标交互面板**:虚拟面板上可配置多个选择目标,支持自定义布局与样式 +- **反应时间记录**:精确记录从面板目标出现到食指选择的反应时长,用于交互性能分析 +- **选择有效性判定**:自动检测食指与目标区域的接触状态,判断选择有效性 +- **动态目标生成**:支持随机/序列生成面板目标,模拟复杂选择任务 + + +## 目录结构 + +``` +HCL_interaction_task/ +├── mobl_arms_index_pointing/ # 食指指向追踪模块 +│ ├── config.yaml # 指向仿真参数配置 +│ ├── simulation.xml # 指向功能模型定义 +│ ├── evaluator.py # 指向模块程序入口 +│ ├── simulator.py # 指向仿真核心逻辑 +│ ├── assets/ # 指向模块模型资源(网格/纹理) +│ └── mujoco_ros_demo/ # 指向模块ROS 2功能包 +├── mobl_arms_index_choice_reaction/ # 面板选择反应模块 +│ ├── config.yaml # 面板交互参数配置 +│ ├── simulation.xml # 面板功能模型定义 +│ ├── main.py # 面板模块程序入口 +│ ├── simulator.py # 面板仿真核心逻辑 +│ └── assets/ # 面板模块模型资源(含面板纹理) +├── README.md # 项目说明文档 +└── requirements.txt # 项目依赖清单 +``` ## 模型结构 -仿真模型定义在`simulation.xml`中,核心组件包括: -- **骨骼结构**:详细的上肢骨骼(锁骨、尺骨、掌骨及指骨等),通过网格文件定义几何形状 -- **标记点(sites)**:用于定位肌肉附着点和关键位置(如手指关节、肌肉路径点) -- **肌腱与肌肉**:定义主要肌肉的路径(三角肌、肱二头肌等)及物理参数,实现逼真驱动 -- **关节**:定义各关节的活动范围和轴方向(如肘关节弯曲角度限制) +### 食指指向模型(`mobl_arms_index_pointing/simulation.xml`) +- 核心组件:上肢骨骼结构、肌肉肌腱系统、目标标记点(小球) +- 关键定义:手指关节活动范围、肌肉附着点、目标位置约束 + +### 面板选择模型(`mobl_arms_index_choice_reaction/simulation.xml`) +- 核心组件:继承基础上肢模型,新增交互面板几何体、多目标区域标记 +- 关键定义:面板碰撞属性、目标区域触发条件、手指接触检测参数 + ## 系统要求 - Ubuntu 22.04 LTS (ROS 2 Humble) -- Python 3.8+(3.10.18) -- MuJoCo 2.3.0+(3.3.7) -- OpenCV -- MediaPipe -- NumPy -- PyYAML -- Conda 环境管理器 +- Python 3.8+(推荐3.10.18) +- MuJoCo 2.3.0+(测试版本3.3.7) +- 依赖库:OpenCV、MediaPipe、NumPy、PyYAML +- 环境管理:Conda 环境管理器 - 额外依赖(C++):libglfw3-dev、libyaml-cpp-dev、libeigen3-dev + ## 安装步骤 -### 1. 克隆仓库 +### 1. 克隆仓库(如适用) ```bash -# 克隆主项目 -git clone https://github.com/yourusername/mobl-arms-index-pointing.git -cd mobl-arms-index-pointing - -# 克隆依赖项目(如需要) -git clone https://github.com/User-in-the-Box/user-in-the-box.git +git clone https://github.com/lbxlb/nn.git +cd HCL_interaction_task ``` -### 2. 安装ROS 2 Humble - -按照官方指南安装:https://docs.ros.org/en/humble/Installation.html - -### 3. 创建并配置环境 +### 2. 环境配置 ```bash # 创建并激活conda环境 conda create -n mjoco_ros python=3.10 conda activate mjoco_ros -# 安装核心依赖 +# 安装依赖 +pip install -r requirements.txt +# 或手动安装 pip install mujoco mediapipe numpy opencv-python pyyaml -# 安装ROS 2相关依赖(如需要) +# 安装ROS 2相关依赖(如使用ROS功能) sudo apt install ros-humble-ros-base - -# 安装项目包 -pip install -e . ``` -### 4. 模型文件准备 +### 3. 模型资源准备 + +1. 下载完整模型文件集(含精细STL模型和纹理): + [完整模型文件集网盘链接](链接: https://pan.baidu.com/s/1sA0BgEPRgxXTqe6ZdEm7Sg?pwd=rq8e 提取码: rq8e) -完整模型文件集(包含40+手部精细模型文件)可通过以下链接下载: -[完整模型文件集网盘链接](https://pan.baidu.com/s/1sA0BgEPRgxXTqe6ZdEm7Sg?pwd=rq8e 提取码: rq8e) +2. 解压资源文件到对应模块的assets目录: + - 指向模块:`mobl_arms_index_pointing/assets/` + - 面板模块:`mobl_arms_index_choice_reaction/assets/` -下载后解压到 `mujoco_ros_demo/config/assets/` 目录 ## 配置说明 -### 仿真参数配置(config.yaml) +### 食指指向模块配置(`mobl_arms_index_pointing/config.yaml`) ```yaml -dt: 0.05 # 仿真步长(越小精度越高,性能消耗越大) -render_mode: "human" # 渲染模式("human"显示窗口,"offscreen"无窗口运行) -resolution: [1280, 960] # 窗口分辨率 [宽度, 高度] -target_pos: [0.4, 0, 0.7] # 食指追踪目标坐标 +dt: 0.05 # 仿真步长 +render_mode: "human" # 渲染模式("human"/"offscreen") +resolution: [1280, 960] # 窗口分辨率 +target_pos: [0.4, 0, 0.7] # 目标小球坐标 ``` +### 面板选择模块配置(`mobl_arms_index_choice_reaction/config.yaml`) + +```yaml +dt: 0.05 # 仿真步长 +render_mode: "human" # 渲染模式 +resolution: [1280, 960] # 窗口分辨率 +panel_pos: [0.5, 0, 0.6] # 面板位置 +target_count: 3 # 面板目标数量 +target_size: 0.05 # 目标区域大小 +reaction_timeout: 5.0 # 反应超时时间(秒) +``` + + ### ROS 2功能包结构 ``` From b12745f2162e15cc573f68ceae262d71e60e435b Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Mon, 22 Dec 2025 08:36:18 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81,?= =?UTF-8?q?=E9=9A=8F=E6=9C=BA=E5=88=B7=E6=96=B0=E9=9D=A2=E6=9D=BF=E9=A2=9C?= =?UTF-8?q?=E8=89=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../simulator.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py index 12a44016ac..769f3f9425 100644 --- a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py @@ -21,6 +21,12 @@ def __init__(self, config_path, model_path): # 1. 模型关键信息+关节range(稳定核心) print("\n📌 模型核心信息:") print(f"关节总数:{self.model.njnt} | 执行器数:{self.model.nu} | qpos长度:{len(self.data.qpos)}") + + # 在__init__里新增:打印所有geom + print("\n📌 所有geom名称:") + for i in range(self.model.ngeom): + geom_name = mujoco.mj_id2name(self.model, mujoco.mjtObj.mjOBJ_GEOM, i) + print(f"geom{i}:{geom_name}") self.key_joints = { 15: { @@ -58,6 +64,25 @@ def __init__(self, config_path, model_path): self.button_touched = {"button-0":False, "button-1":False, "button-2":False, "button-3":False} self.finger_geom_name = "hand_2distph" + # 新增:找到screen对应的geom ID(控制面板颜色) + self.screen_geom_name = "screen" + self.screen_geom_id = mujoco.mj_name2id( + self.model, mujoco.mjtObj.mjOBJ_GEOM, self.screen_geom_name + ) + if self.screen_geom_id == -1: + print(f"⚠️ 未找到面板geom({self.screen_geom_name}),颜色控制失效") + else: + print(f"✅ 找到面板geom(ID:{self.screen_geom_id}),已启用颜色切换") + + # 新增:面板要切换的4种颜色(和按钮颜色对应) + self.screen_colors = [ + [0.8, 0.1, 0.1, 1.0], # 红色(对应button-0) + [0.1, 0.8, 0.1, 1.0], # 绿色(对应button-1) + [0.1, 0.1, 0.8, 1.0], # 蓝色(对应button-2) + [0.8, 0.8, 0.1, 1.0] # 黄色(对应button-3) + ] + self.color_switch_interval = 100 # 每隔100步切换一次颜色(≈3秒) + def reset(self): """重置关节到range中间值""" mujoco.mj_resetData(self.model, self.data) @@ -75,6 +100,22 @@ def step(self): if not self.is_running: return self.data.qpos.copy() + # 新增:每隔N步随机切换面板颜色(兼容所有MuJoCo版本) + if self.screen_geom_id != -1 and self.current_step % self.color_switch_interval == 0: + # 随机选一种颜色 + random_color = self.screen_colors[np.random.randint(0, len(self.screen_colors))] + # 关键修改:直接赋值model.geom_rgba(替代过时的mj_geom_rgba) + self.model.geom_rgba[self.screen_geom_id] = random_color + # 打印颜色变化 + color_map = { + tuple(self.screen_colors[0]): "红", + tuple(self.screen_colors[1]): "绿", + tuple(self.screen_colors[2]): "蓝", + tuple(self.screen_colors[3]): "黄" + } + color_name = color_map[tuple(random_color)] + print(f"\n🎨 step{self.current_step} 面板颜色切换为:{color_name}色") + # 1. 关节慢速摆动(不超range) for jnt_id, info in self.key_joints.items(): if jnt_id >= len(self.data.qpos): From 2ebdf124db6538c4a844646b49b8d745f08eabfc Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Mon, 22 Dec 2025 10:58:26 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E6=B7=BB=E5=8A=A0task=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E5=A5=96=E5=8A=B1=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config.yaml | 8 +- .../mobl_arms_index_choice_reaction/main.py | 59 ++++++++++++- .../simulator.py | 12 +-- .../mobl_arms_index_choice_reaction/task.py | 82 +++++++++++++++++++ 4 files changed, 152 insertions(+), 9 deletions(-) create mode 100644 src/HCL_interaction_task/mobl_arms_index_choice_reaction/task.py diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml index 74b0cdcb77..104edc1f2e 100644 --- a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/config.yaml @@ -1,2 +1,8 @@ +# 原有配置 dt: 0.01 # 仿真步长(调慢一点,看得清) -render_mode: "human" # 开启可视化! \ No newline at end of file +render_mode: "human" # 开启可视化! + +target_button: 0 # 目标按钮(可选0/1/2/3,对应红/绿/蓝/黄) +button_reward: 10.0 # 选对按钮的奖励值 +effort_cost: 0.01 # 动作成本扣分系数(模型有执行器时生效) +timeout: 600 # 任务超时步数(300步≈9秒,可根据需要调整) \ No newline at end of file diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/main.py b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/main.py index 6f05e6172a..affc1d45e5 100644 --- a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/main.py +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/main.py @@ -1,13 +1,68 @@ +import mujoco from simulator import IndexSimulator +from task import ChoicePanelTask # 新增:导入任务类 +import yaml # 新增:加载配置文件 def main(): # 1. 配置路径 config_path = "config.yaml" model_path = "simulation.xml" - # 2. 初始化仿真器+运行 + # 新增:加载配置文件(给任务传参) + with open(config_path, 'r', encoding='utf-8') as f: + config = yaml.safe_load(f) + + # 2. 初始化仿真器+任务实例 sim = IndexSimulator(config_path, model_path) - sim.run_simulation() # 所有打印都在run_simulation里,这里不用额外打印 + task = ChoicePanelTask(config, sim) # 关联配置和仿真器 + + # 3. 重置仿真和任务 + sim.reset() + task.reset() + + # 4. 仿真循环(整合任务逻辑,替代sim.run_simulation()) + try: + # 先启动可视化viewer(和原仿真器逻辑一致) + try: + sim.viewer = mujoco.viewer.launch_passive(sim.model, sim.data) + # 优化视角 + sim.viewer.cam.azimuth = 135 + sim.viewer.cam.elevation = -15 + sim.viewer.cam.distance = 0.6 + sim.viewer.cam.lookat = [0.45, -0.15, 0.8] + print("✅ 可视化窗口启动成功!能看到手臂慢速摆动") + except Exception as e: + print(f"⚠️ 可视化启动失败(不影响动作):{e}") + print("📢 无窗口模式:终端打印关节角度,确认动作真实发生!") + + # 循环执行仿真和任务更新 + while sim.is_running: + # 仿真步进(模型运动) + sim.step() + # 任务状态更新(判断成败、计算奖励) + task_status = task.update() + + # 如果任务完成(成功/超时),重置仿真和任务 + if task_status["done"]: + print("\n🔄 准备开始新一轮任务...") + sim.reset() + task.reset() + + # 渲染同步(保持窗口流畅) + if sim.viewer: + sim.viewer.sync() + time.sleep(0.001) + + except KeyboardInterrupt: + print("\n\n⚠️ 检测到Ctrl+C,正在优雅退出仿真...") + sim.is_running = False + finally: + # 关闭资源 + sim.close() + print(f"\n✅ 仿真正常结束!共运行{sim.current_step}步") + +# 新增:导入time(用于渲染延时) +import time if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py index 769f3f9425..fae865d76a 100644 --- a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/simulator.py @@ -32,20 +32,20 @@ def __init__(self, config_path, model_path): 15: { "name": "shoulder_rot (肩关节旋转)", "range": (-1.57, 0.349), - "step": 0.02, - "current_step_dir": 0.02 # 运动方向 + "step": 0.05, + "current_step_dir": 0.05 # 运动方向 }, 16: { "name": "elbow_flexion (肘关节弯曲)", "range": (0, 2.26893), - "step": 0.03, - "current_step_dir": 0.03 + "step": 0.08, + "current_step_dir": 0.08 }, 17: { "name": "pro_sup (前臂旋前旋后)", "range": (-1.5708, 1.5708), - "step": 0.015, - "current_step_dir": 0.015 + "step": 0.04, + "current_step_dir": 0.04 } } diff --git a/src/HCL_interaction_task/mobl_arms_index_choice_reaction/task.py b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/task.py new file mode 100644 index 0000000000..a8a7bff0a0 --- /dev/null +++ b/src/HCL_interaction_task/mobl_arms_index_choice_reaction/task.py @@ -0,0 +1,82 @@ +import numpy as np + +class ChoicePanelTask: + def __init__(self, config, sim): + self.config = config + self.sim = sim # 关联你的仿真器实例 + # 从配置中读取任务参数(后续在config.yaml中配置) + self.target_button = config.get("target_button", 0) # 默认目标按钮0 + self.button_reward = config.get("button_reward", 10.0) # 选对奖励 + self.effort_cost = config.get("effort_cost", 0.01) # 动作成本系数 + self.timeout_steps = config.get("timeout", 300) # 超时步数 + + self.step_count = 0 # 记录当前任务步数 + self.done = False # 任务是否完成 + self.reward = 0.0 # 当前奖励值 + + def reset(self): + """重置任务状态(每次仿真/任务重新开始时调用)""" + self.step_count = 0 + self.done = False + self.reward = 0.0 + print(f"\n🔄 任务已重置!目标按钮:button-{self.target_button},超时步数:{self.timeout_steps}") + return {"reward": self.reward, "done": self.done} + + def update(self): + """每一步仿真都调用,更新任务状态(核心逻辑)""" + if self.done: # 任务已完成,直接返回状态 + return {"reward": self.reward, "done": self.done} + + self.step_count += 1 + self.reward = 0.0 + + # 1. 检测:指尖是否碰到目标按钮 + is_touch_target = self._check_button_contact(self.target_button) + if is_touch_target: + self.reward += self.button_reward # 选对按钮,添加奖励 + self.done = True + print(f"\n🎉 任务成功!指尖碰到button-{self.target_button},奖励:{self.reward:.2f}") + + # 2. 动作成本扣分(兼容执行器为0的情况) + ctrl_effort = np.sum(np.abs(self.sim.data.ctrl)) if self.sim.model.nu > 0 else 0.0 + effort_penalty = self.effort_cost * ctrl_effort + self.reward -= effort_penalty + if self.sim.model.nu == 0 and self.step_count % 50 == 0: + print(f"\n💡 提示:当前模型无执行器,动作成本扣分暂不生效(后续添加执行器后自动启用)") + + # 3. 超时判断:超过步数未完成则任务失败 + if self.step_count >= self.timeout_steps: + self.done = True + self.reward -= 5.0 # 超时额外扣分 + print(f"\n⏰ 任务超时失败!未在{self.timeout_steps}步内碰到button-{self.target_button},最终奖励:{self.reward:.2f}") + + # 4. 定期打印任务状态(每50步) + if self.step_count % 50 == 0 and not self.done: + print(f"\n💰 step{self.step_count} 任务状态:") + print(f" 是否碰到目标按钮:{'是' if is_touch_target else '否'}") + print(f" 当前奖励:{self.reward:.2f}") + print(f" 剩余步数:{self.timeout_steps - self.step_count}") + + return {"reward": self.reward, "done": self.done} + + def _check_button_contact(self, button_id): + """检测指尖(hand_2distph)是否碰到指定按钮(button-x)""" + finger_geom_name = self.sim.finger_geom_name # 从仿真器获取指尖geom名称 + target_button_name = f"button-{button_id}" # 目标按钮geom名称 + + # 遍历所有碰撞对,判断是否包含「指尖」和「目标按钮」 + for i in range(self.sim.data.ncon): + contact = self.sim.data.contact[i] + # 获取两个碰撞geom的名称 + geom1_id = contact.geom1 + geom2_id = contact.geom2 + geom1_name = mujoco.mj_id2name(self.sim.model, mujoco.mjtObj.mjOBJ_GEOM, geom1_id) + geom2_name = mujoco.mj_id2name(self.sim.model, mujoco.mjtObj.mjOBJ_GEOM, geom2_id) + + # 判断是否是「指尖-目标按钮」的碰撞 + if (finger_geom_name in [geom1_name, geom2_name]) and (target_button_name in [geom1_name, geom2_name]): + return True + return False + +# 导入mujoco(避免内部函数报错) +import mujoco \ No newline at end of file From 36c113f5a65cc96ecc5139c70bceaf3cb4a847bf Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Mon, 22 Dec 2025 11:23:29 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E6=9B=B4=E6=96=B0README.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HCL_interaction_task/README.md | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/HCL_interaction_task/README.md b/src/HCL_interaction_task/README.md index 806c611cfb..bacbb96ca0 100644 --- a/src/HCL_interaction_task/README.md +++ b/src/HCL_interaction_task/README.md @@ -30,6 +30,9 @@ - **反应时间记录**:精确记录从面板目标出现到食指选择的反应时长,用于交互性能分析 - **选择有效性判定**:自动检测食指与目标区域的接触状态,判断选择有效性 - **动态目标生成**:支持随机/序列生成面板目标,模拟复杂选择任务 +- **任务逻辑闭环**:通过task.py实现奖励机制、超时判断、任务重置的完整逻辑闭环 +- **友好状态提示**:终端实时输出任务进度、奖励数值、碰撞状态,便于调试与分析 + ## 目录结构 @@ -48,6 +51,7 @@ HCL_interaction_task/ │ ├── simulation.xml # 面板功能模型定义 │ ├── main.py # 面板模块程序入口 │ ├── simulator.py # 面板仿真核心逻辑 +│ ├── task.py # 面板任务逻辑核心 │ └── assets/ # 面板模块模型资源(含面板纹理) ├── README.md # 项目说明文档 └── requirements.txt # 项目依赖清单 @@ -65,6 +69,41 @@ HCL_interaction_task/ - 关键定义:面板碰撞属性、目标区域触发条件、手指接触检测参数 + +## 核心模块说明 + +### 面板任务逻辑核心(`mobl_arms_index_choice_reaction/task.py`) +#### 1. 模块作用 +作为面板选择反应模块的「大脑」,负责串联仿真流程与交互逻辑,实现从任务初始化到结果判定的完整闭环,无需侵入修改仿真核心代码,保持模块解耦性。 + +#### 2. 核心功能 +| 功能项 | 详细描述 | +|-----------------------|--------------------------------------------------------------------------| +| 任务初始化/重置 | 支持任务参数加载与状态重置,实现多轮任务自动循环运行 | +| 指尖-目标碰撞检测 | 精准识别食指远节(`hand_2distph`)与面板目标(`button-x`)的接触状态 | +| 奖励机制设计 | 正确选择目标奖励(默认10.0)、超时失败扣分(默认-5.0)、动作成本惩罚(可选) | +| 超时判断 | 基于配置的超时步数/时间,自动判定任务失败并触发重置 | +| 实时状态反馈 | 每50步输出任务进度、剩余步数、当前奖励,便于实时监控仿真过程 | +| 兼容无执行器模型 | 对无肌肉/执行器驱动的模型提供友好提示,不影响任务核心逻辑运行 | + +#### 3. 关键类与方法 +- **`ChoicePanelTask` 类**:任务逻辑封装核心,关联仿真器实例与配置参数 + - `__init__`:初始化任务参数(目标按钮、奖励值、超时步数) + - `reset()`:重置任务状态,用于多轮任务循环 + - `update()`:每步仿真调用,更新任务状态(碰撞检测、奖励计算、超时判断) + - `_check_button_contact()`:内部核心方法,实现指尖与目标的碰撞检测 + +#### 4. 配置依赖 +任务参数完全读取自`config.yaml`,支持灵活配置,无需修改代码: +```yaml +# 任务相关配置(已整合到config.yaml) +target_button: 0 # 目标按钮ID(0-3对应红/绿/蓝/黄) +button_reward: 10.0 # 正确选择奖励值 +effort_cost: 0.01 # 动作成本惩罚系数(有执行器时生效) +timeout: 600 # 任务超时步数(对应仿真时长) +``` + + ## 系统要求 - Ubuntu 22.04 LTS (ROS 2 Humble) @@ -131,6 +170,11 @@ panel_pos: [0.5, 0, 0.6] # 面板位置 target_count: 3 # 面板目标数量 target_size: 0.05 # 目标区域大小 reaction_timeout: 5.0 # 反应超时时间(秒) +# 任务逻辑配置(新增) +target_button: 0 # 目标按钮ID(0-3) +button_reward: 10.0 # 正确选择奖励值 +effort_cost: 0.01 # 动作成本惩罚系数 +timeout: 600 # 任务超时步数 ``` From 52f956a9c8d55aed4b92bb70f0a118938b0926c1 Mon Sep 17 00:00:00 2001 From: lbxlb <3378337402@qq.com> Date: Mon, 22 Dec 2025 23:02:32 +0800 Subject: [PATCH 8/8] =?UTF-8?q?=E4=B8=8A=E4=BC=A0requirements.txt,?= =?UTF-8?q?=E7=BB=99=E5=87=BA=E4=BE=9D=E8=B5=96=E6=B8=85=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/HCL_interaction_task/requirements.txt | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/HCL_interaction_task/requirements.txt diff --git a/src/HCL_interaction_task/requirements.txt b/src/HCL_interaction_task/requirements.txt new file mode 100644 index 0000000000..336a65dde1 --- /dev/null +++ b/src/HCL_interaction_task/requirements.txt @@ -0,0 +1,18 @@ +# 基础依赖 +python >= 3.8 +numpy >= 1.21.0 +pyyaml >= 6.0.0 + +# 计算机视觉与手势追踪 +opencv-python >= 4.5.5.64 +mediapipe >= 0.8.10 + +# 物理仿真 +mujoco >= 2.3.0 + +# ROS 2 Python客户端 +rclpy >= 3.3.0 # 对应ROS 2 Humble版本 + +# 可选:可视化工具(根据项目实际使用情况添加) +matplotlib >= 3.5.0 +plotly >= 5.5.0 \ No newline at end of file