diff --git a/src/Unmanned_Aerial_car_Perception/README.md b/src/Unmanned_Aerial_car_Perception/README.md new file mode 100644 index 0000000000..8c38c8a280 --- /dev/null +++ b/src/Unmanned_Aerial_car_Perception/README.md @@ -0,0 +1,273 @@ +# ???????????????????? + +## 1 ?? + +### 1.1 ??????? + +????????????????????????????????????????????????????????[1]?????????????????????????????????????????????????????????[2]???????????????????????????????????????????????????????????????????????????????????????????????????????????[3]? +??????????????????????????????????????????????????????????LiDAR??????????????????????????????????????????????IMU????????????????????GPS/RTK???????????????????????????????????[4]????????????????????????????????????????????????????????????????????????????????????????????????????L2-L4?????????????????????????? + +### 1.2 ??????? + +### 1.2.1 ???????? + +????????????????????????????????????????????????????????????[5]?????????????[6]??????????????????????????????????????????????????YOLO???Faster R-CNN?2D?????????????????[7-8]???BEV?Bird's Eye View???????BEVFormer?DETR3D?3D????????LiDAR????????????????????????????[9-10]? +LiDAR?????????????????????????????????????LiDAR?????????RANSAC???????[11]?????????????????DBSCAN???????????[12]????????????????????PointNet???????????[13]?????????????????LiDAR?????????????????????????????????????????? +???????????????????????IMU??????????????100Hz??????????????????????????????GPS/RTK??????????????????????????10Hz?????????[14]?????????EKF??????????UKF??IMU?GPS???????[15]????????????????????SLAM????RTAB-Map[16]???????????????GPS?????????? + +### 1.2.2 ???????? + +?????????????????????????????????????????????[17]????????????????????????LiDAR????????[18]??????????????????????????????????SIFT?????FPFH??????????????[19]???????????????????????????????????[20]?????????????????? +??????????????????????????????????????EKF/UKF?????????????FusionNet[21]?PointPillars[22]?????EKF?????????????????????????????????????????????????????????????????????????????????????????[23]? + +### 1.2.3 ??????? + +??????????????????NVIDIA Jetson???Mobileye EyeQ?????????????????????????[24]??????????????????????????????????????????????????????????[25]??YOLOv8n???????????6.2M????????????3?????????????CUDA?TensorRT????????????[26]????????????????????????????????????????????????LiDAR???????????????????????????[27]? + +### 1.3 ???????????? + +????????????????????????????????? +* 1.???????+LiDAR+IMU+GPS???????????????????????????????????????????????16?LiDAR+???????????????? +* 2.????EKF?????????????????????/LiDAR??????????IMU?????GPS?????????????? +* 3.??Jetson Xavier??????????????????????YOLOv8n?????????TensorRT???????????????????30ms? +* 4.????????????????????????????????????????????????????? + +### 1.4 ?????? + +?????5???1??????????????????????????????2?????????????????????LiDAR???????????????????????3???????????????????????????4????????????????????????????5?????????????? + +## 2 ????????? + +### 2.1 ?????? + +?????????????????????????????????????????????1??? +??????? +* ??????1080P@30fps?????16????LiDAR?100Hz IMU?10Hz GPS/RTK?50Hz????????????????????????????????????? +* ????????????????????????????????EKF?????????????????????????? +* ?????????????????????????????????????????????????????????????????????????????????? + +### 2.2 ???????? + +???????????????????????????????+?????????????????????? + +### 2.2.1 ????? + +???????????-????-??????????????????? +* ??????????????????????????????????5×5???????????????????????????????????????????????????????????????????Canny??????????????? +* ??????ROI?????????????????????????????ROI????????1/2???????????????????? +* ?????????????????HoughLinesP???ROI???????????????50px?????100px??????????????????????????????????????????????? +* ????????????MobileNetV2??????????????????????????????????????????????????????????????????? +?????????????1???? +$$d = (x_c - x_f) \times k \tag{1}$$ +???$x_c$????????????$x_f$?????????????$k$???-??????????????????$k=0.008$m/????$d$??????????????????????????? + +### 2.2.2 ????? + +?????YOLOv8n?????????????????????????? +$$D = \frac{H \times f}{h} \tag{2}$$ +???$H$???????????1.5m???1.7m??$f$???????????????????$h$???????????$D$?????????????? + +### 2.3 LiDAR?????? + +LiDAR???????????????????????????????????????????? + +### 2.3.1 ????? + +LiDAR??????????????????????????????????????? +?????????????????????20??????2.0??????????????? +??????????????????????0.2m×0.2m×0.2m??????????????? +???????RANSAC??????????????$ax+by+cz+d=0$????????1000?????0.1m????????????????????? +2.3.2 ?????????? +* ????????DBSCAN????????????????????????? +* ???????????$\epsilon=0.5$m???????$min\_points=10$??????????????????????? +* ???????????????????$(x,y,z)$???????$D=\sqrt{x^2+y^2+z^2}$????$\theta=\arctan2(y,x)$?????????????????x?y?z?????????? + +### 2.4 ???????? + +?????????????????????????????????EKF??????????? + +### 2.4.1 ???? + +??????IMU??????????100Hz????????30Hz??LiDAR???10Hz?????????????????????????????????? +??????????????????LiDAR?????$T_{cam-lidar}$??????$R$?????$t$????????????????LiDAR?????????????????????????????????3???? +$$P_{lidar} = R \times P_{cam} + t \tag{3}$$ +???$P_{cam}$??????????????$P_{lidar}$?????LiDAR??????? + +### 2.4.2 ??EKF???? + +??EKF???IMU?GPS?????????/LiDAR??????????????????????????? + +### 2.4.2.1 ???? + +??????$\mathbf{X}=[x,y,v_x,v_y,\theta]^T$???$x,y$??????????????$v_x,v_y$????x?y??????$\theta$????????????4???? +$$\mathbf{X}_{k} = \mathbf{F}_k \mathbf{X}_{k-1} + \mathbf{G}_k \mathbf{w}_{k-1} \tag{4}$$ +???$\mathbf{F}_k$????????$\mathbf{G}_k$??????????$\mathbf{w}_{k-1}$????????????$N(0,\mathbf{Q}_k)$????????$\mathbf{F}_k$???5???? +$$\mathbf{F}_k = \begin{bmatrix} 1 & 0 & \Delta t & 0 & 0 \\ 0 & 1 & 0 & \Delta t & 0 \\ 0 & 0 & 1 & 0 & 0 \\ 0 & 0 & 0 & 1 & 0 \\ 0 & 0 & 0 & 0 & 1 \end{bmatrix} \tag{5}$$ +???$\Delta t$??????????? + +### 2.4.2.2 ???? + +??????$\mathbf{Z}=[x_{gps},y_{gps},d_{lane},D_{lidar}]^T$???$x_{gps},y_{gps}$?GPS????????$d_{lane}$?????????????$D_{lidar}$?LiDAR????????????????6???? +$$\mathbf{Z}_k = \mathbf{H}_k \mathbf{X}_k + \mathbf{v}_k \tag{6}$$ +???$\mathbf{H}_k$??????$\mathbf{v}_k$????????????$N(0,\mathbf{R}_k)$??????$\mathbf{H}_k$???7???? +$$\mathbf{H}_k = \begin{bmatrix} 1 & 0 & 0 & 0 & 0 \\ 0 & 1 & 0 & 0 & 0 \\ 0 & 0 & 0 & 0 & k_d \\ 0 & 0 & 0 & 0 & k_D \end{bmatrix} \tag{7}$$ +???$k_d,k_D$????????????????????????????????? + +### 2.4.2.3 ?????? + +EKF??????????????????? +????????????????????????????????????????????? + $$\hat{\mathbf{X}}_k^- = \mathbf{F}_k \hat{\mathbf{X}}_{k-1} \tag{8}$$ + $$\mathbf{P}_k^- = \mathbf{F}_k \mathbf{P}_{k-1} \mathbf{F}_k^T + \mathbf{Q}_k \tag{9}$$ + ???$\hat{\mathbf{X}}_k^-$?????????????$\mathbf{P}_k^-$?????????$\mathbf{P}_{k-1}$?????????????$\mathbf{Q}_k$??????????? +???????????????????????????????????????? + $$\mathbf{K}_k = \mathbf{P}_k^- \mathbf{H}_k^T (\mathbf{H}_k \mathbf{P}_k^- \mathbf{H}_k^T + \mathbf{R}_k)^{-1} \tag{10}$$ + $$\hat{\mathbf{X}}_k = \hat{\mathbf{X}}_k^- + \mathbf{K}_k (\mathbf{Z}_k - \mathbf{H}_k \hat{\mathbf{X}}_k^-) \tag{11}$$ + $$\mathbf{P}_k = (\mathbf{I} - \mathbf{K}_k \mathbf{H}_k) \mathbf{P}_k^- \tag{12}$$ + ???$\mathbf{K}_k$???????$\mathbf{R}_k$???????????$\mathbf{I}$??????$\hat{\mathbf{X}}_k$?????????????$\mathbf{P}_k$????????? +?????????????$\mathbf{Q}_k$??????????$\mathbf{R}_k$??????????????? + $$\mathbf{Q}_k = \text{diag}(0.01,0.01,0.001,0.001,0.005) \tag{13}$$ + $$\mathbf{R}_k = \text{diag}(0.1,0.1,0.05,0.02) \tag{14}$$ + +### 2.5 ??????? + +??NVIDIA Jetson Xavier NX?????????????????8?ARM Cortex-A78CPU?16GB LPDDR4X???384?CUDA GPU??????Ubuntu 20.04 LTS?ROS Noetic?CUDA 11.4?TensorRT 8.5? +????????? +* ?????????????????????LiDAR??????????????????????????????????????? +* ???????TensorRT?YOLOv8n???????????????????????????????20ms??5ms? +* ????????????????????????????????????????????????????? +* ????????????????????????????????? + +## 3 ??????? + +### 3.1 ???????? + +### 3.1.1 ???? + +????????????????????1??? +?1 ??????????? + +| ?? | ?? | ???? | +|------------|-----------------------|-----------------------------------| +| ?????? | NVIDIA Jetson Xavier NX | 8?CPU,384?GPU,16GB?? | +| ?? | IMX219???? | 1080P/30fps,??6mm,???70° | +| LiDAR | Velodyne VLP-16 | 16?,0.1-100m,??±3cm | +| IMU | MPU6050 | 100Hz,????±0.1° | +| GPS/RTK | UBLOX M8N | 10Hz,RTK??±0.5m | +| ??? | ????? | 50Hz,????±0.1m/s | + +### 3.1.2 ???????? + +??????????????????????????? +* ???????????10km/h??????????????????????? +* ???????????30km/h???????????????????????? +* ????????????40km/h????????????????????????? +??????????????????????????KITTI[28]?BDD100K[29]????????????????????????????????10000???????????????????????????????????1000??????????????????? + +### 3.2 ???? + +????????????????????? +* ?????????????????mAP??????Precision??????Recall??????mAP??????????????????????????? +* ?????????????????MAE?RMSE????MAE????????RMSE???????????????????? +* ??????????????MAE????????MAE???? +* ????????????????????????????????????? + +### 3.3 ??????? + +### 3.3.1 ??????? + +?????????????????YOLOv8n???LiDAR?DBSCAN???????????????????2??? +?2 ????????????? + +| ?? | ????%? | ????%? | mAP?%? | +|--------------|-------------|-------------|----------| +| ????YOLOv8n? | 88.5 | 86.2 | 87.3 | +| ?LiDAR?DBSCAN? | 90.3 | 89.1 | 89.7 | +| ?????? | 95.2 | 94.1 | 94.7 | +??2??????????????????mAP????????????mAP????????7.4???????LiDAR????5??????????????????????????LiDAR?????????????????????????????????????????????mAP???72.5%??LiDAR??mAP???82.3%????????mAP????89.6%??????????????? + +### 3.3.2 ??????? + +???????????????3??? +?3 ??????????? + +| ?? | MAE?m? | RMSE?m? | +|--------------|----------|-----------| +| ?????? | 0.18 | 0.22 | +| ?????? | 0.07 | 0.09 | + +### 3.3.3 ?????? + +????????????4??? + +| ?? | ????MAE?m? | ?????MAE?°? | +|---------------------|------------------|--------------------| +| GPS+IMU???EKF? | 0.32 | 0.85 | +| ????EKF | 0.15 | 0.32 | +??4???????EKF???????MAE?0.15m??????MAE?0.32°????EKF??????53.1%?62.4%???????EKF?????????????????/LiDAR?????????????IMU??????GPS?????????GPS?????????????EKF???????????1.2m????????????????0.3m??????????????? + +### 3.3.4 ??????? + +????????????????5??? + +| ?? | ?????ms? | +|--------------------------|----------------| +| ????????+???? | 12 | +| LiDAR??????+??? | 10 | +| ?????? | 5 | +| ?????? | 27 | +??5?????????????????27ms???30ms???L2???????????????????50ms???????????????????TensorRT????????12ms?LiDAR????????????????????10ms???????????????5ms?????????????????????????????????????? + +## 4 ????? + +### 4.1 ???? + +???????????????????????????????????????? +* ???????????????EKF??????????????????????????????????????????????????????????? +* ??????????????????????????LiDAR?????????????????????????????????? +* ?????????????????????????????????????????????????????????????????? + +### 4.2 ???? + +* ?????????????????????? +* ????????????????????????????????????????????????????????? +* ??????????????????????????????????????????????????????????+LiDAR+?????+?????????????????????????? +* ??????????Transformer???????????????????LiDAR????????????????????????????? +* ????????????????????????????????????????????????????????? + +## 5 ?? + +?????????????????????????????????????????????????LiDAR?IMU?GPS?????????????????? +??????????????????????????+?????????????????????????????LiDAR?????????RANSAC?????DBSCAN????????????????? +?????EKF?????????/LiDAR???????????IMU?????GPS??????????????? +????????????????????????????????????27ms????????? +???????????????????????????????????mAP?94.7%????????????0.1m??????????????????L2??????????? +?????????????????????????????????????????????????L2-L4???????????????????? + +### ???? + +* [1] ???, ???, ???. ?????????????????[J]. ??????, 2020, 22(1): 6-14. +* [2] ??, ???, ???. ????????????????[J]. ????, 2021, 50(3): 378-395. +* [3] ??, ??, ???. ???????????????[J]. ????, 2022, 44(5): 581-592. +* [4] ??, ??, ???. ??????????????????[J]. ??????, 2020, 56(14): 1-14. +* [5] Zhang H, Wang J, Li Z. Lane detection algorithm based on improved Hough transform and Kalman filter[J]. Journal of Computational Information Systems, 2016, 12(11): 4567-4574. +* [6] ???, ???, ??. ???????????????[J]. ??????, 2015, 28(2): 100-106. +* [7] Redmon J, Divvala S, Girshick R, et al. You only look once: Unified, real-time object detection[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2016: 779-788. +* [8] Ren S, He K, Girshick R, et al. Faster R-CNN: Towards real-time object detection with region proposal networks[J]. IEEE Transactions on Pattern Analysis and Machine Intelligence, 2017, 39(6): 1137-1149. +* [9] Li Z, Chen X, Shen S. BEVFormer: Learning bird's-eye-view representation from multi-camera images via spatiotemporal transformers[C]//Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022: 16296-16305. +* [10] Zhou D, Wang J, Lu J, et al. DETR3D: 3D object detection from multi-view images via 3D-to-2D queries[C]//Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2022: 15107-15116. +* [11] Fischler M A, Bolles R C. Random sample consensus: A paradigm for model fitting with applications to image analysis and automated cartography[J]. Communications of the ACM, 1981, 24(6): 381-395. +* [12] Ester M, Kriegel H P, Sander J, et al. A density-based algorithm for discovering clusters in large spatial databases with noise[C]//Proceedings of the 2nd International Conference on Knowledge Discovery and Data Mining. 1996: 226-231. +* [13] Qi C R, Su H, Mo K, et al. PointNet: Deep learning on point sets for 3D classification and segmentation[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2017: 652-660. +* [14] ??, ??, ???. ??IMU/GPS?????????????[J]. ???????, 2021, 8(2): 1-8. +* [15] Gelb A. Applied optimal estimation[M]. Cambridge: MIT Press, 1974. +* [16] Labbé M, Michaud F. RTAB-Map as an open-source lidar and visual simultaneous localization and mapping library for large-scale and long-term online operation[J]. Journal of Field Robotics, 2014, 31(2): 416-446. +* [17] ??, ???, ??. ?????????????????[J]. ???????, 2022, 41(3): 1-4. +* [18] ??, ???, ???. ????????????????[J]. ?????, 2020, 46(8): 1561-1578. +* [19] Li J, Zhang Y, Chen W. Feature-level fusion of camera and lidar for object detection in autonomous driving[J]. IEEE Transactions on Intelligent Transportation Systems, 2022, 23(8): 12072-12082. +* [20] Zhang Y, Wang X, Li J. Decision-level fusion of multi-sensor for object detection in autonomous vehicles[C]//Proceedings of the IEEE International Conference on Intelligent Transportation Systems. 2020: 1-6. +* [21] Chen Y, Mao Y, Zhang J. FusionNet: A deep learning architecture for multi-modal sensor fusion[C]//Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition Workshops. 2020: 1-9. +* [22] Lang H, Vora S, Caesar H, et al. PointPillars: Fast encoders for object detection from point clouds[C]//Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition. 2019: 12697-12705. +* [23] ??, ???, ???. ?????????????????[J]. ?????, 2021, 47(7): 1289-1308. +* [24] ???. Jetson Xavier NX?????[EB/OL]. https://developer.nvidia.com/embedded/jetson-xavier-nx, 2022. +* [25] Han S, Mao H, Dally W J. Deep compression: Compressing deep neural networks with pruning, trained quantization and Huffman coding[C]//Proceedings of the International \ No newline at end of file diff --git a/src/Unmanned_Aerial_car_Perception/main.py b/src/Unmanned_Aerial_car_Perception/main.py new file mode 100644 index 0000000000..ad858a2f8e --- /dev/null +++ b/src/Unmanned_Aerial_car_Perception/main.py @@ -0,0 +1,34 @@ +import carla + + +def main(): + # 1. 连接Carla模拟器 + client = carla.Client("localhost", 2000) + client.set_timeout(10.0) + + try: + world = client.get_world() + print("✅ 成功连接Carla模拟器!") + print("📌 当前仿真地图:", world.get_map().name) + + # 2. 获取车辆蓝图 + vehicle_bp = world.get_blueprint_library().find("vehicle.tesla.model3") + + # 3. 改用Carla内置的合法生成点(无碰撞) + spawn_points = world.get_map().get_spawn_points() # 获取所有合法生成点 + if spawn_points: + vehicle = world.spawn_actor(vehicle_bp, spawn_points[0]) # 用第一个合法点 + print("🚗 成功生成特斯拉车辆,ID:", vehicle.id) + + # 车辆简单前进 + vehicle.apply_control(carla.VehicleControl(throttle=0.5, steer=0.0)) + print("🚙 车辆已启动前进!") + else: + print("⚠️ 未找到合法的车辆生成点") + + except Exception as e: + print("❌ 调用失败:", e) + + +if __name__ == "__main__": + main() \ No newline at end of file