Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
509fdc0
init project
Z-w-7799 Sep 8, 2025
28c7fb2
提高行人安全模块成功
Z-w-7799 Sep 9, 2025
df1fa59
提高行人安全项目创建成功
Z-w-7799 Sep 9, 2025
4bde144
更改项目名称
Z-w-7799 Sep 9, 2025
f14ddea
Rename project to enhance_pedestrian_safety
Z-w-7799 Sep 9, 2025
18d53ae
封装第三题至函数中 补充第二题函数调用
Z-w-7799 Sep 22, 2025
e57b29d
Merge remote-tracking branch 'origin/main'
Z-w-7799 Oct 8, 2025
af29be8
添加我的配置需求
Z-w-7799 Dec 1, 2025
d56a5b2
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 15, 2025
c251745
提交基础生成摄像头文件
Z-w-7799 Dec 15, 2025
dcb9f92
Merge remote-tracking branch 'origin/main' into main
Z-w-7799 Dec 15, 2025
15b41f0
合并远程更新并解决冲突
Z-w-7799 Dec 15, 2025
20700a3
提交说明:删除了一些非必要文件 写入main.py代码 数据图片以网盘链接附于readme文件中
Z-w-7799 Dec 15, 2025
66183f3
多传感器、多天气、多时间的数据收集优化
Z-w-7799 Dec 15, 2025
21c0362
行人安全专用场景,四种摄像头布局,生成过马路的重点,确保数据有价值
Z-w-7799 Dec 16, 2025
03bd323
解决冲突
Z-w-7799 Dec 16, 2025
becea13
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
55378e8
行人场景合理性优化,摄像头数据收集优化,修复错误
Z-w-7799 Dec 16, 2025
ef7611c
Merge remote-tracking branch 'origin/main' into main
Z-w-7799 Dec 16, 2025
a4ff7f0
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
32bc331
将绝对路径修改为相对
Z-w-7799 Dec 16, 2025
754019e
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
0276b7b
分离配置carla部分,添加CARLA 工具模块,添加固定路口监控
Z-w-7799 Dec 16, 2025
91a6298
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
c410b93
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
0beaa44
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
bf975b3
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 16, 2025
7e5b23b
拼接图像,添加智能行人行为控制
Z-w-7799 Dec 16, 2025
f4edd22
Merge remote-tracking branch 'origin/main'
Z-w-7799 Dec 16, 2025
a6f4e4c
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 17, 2025
8f5db2c
新增配置管理模块和日志模块 进行模块化重构
Z-w-7799 Dec 16, 2025
571ca9e
Merge branch 'main' of github.com:Z-w-7799/nn
Z-w-7799 Dec 17, 2025
2d0b4ed
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 17, 2025
d2b7fa1
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 17, 2025
6816cd8
新增了自动标注系统和数据验证系统
Z-w-7799 Dec 17, 2025
ec76682
Merge branch 'OpenHUTB:main' into main
Z-w-7799 Dec 17, 2025
d367e07
新增场景管理器和数据分析器
Z-w-7799 Dec 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
337 changes: 337 additions & 0 deletions src/enhance_pedestrian_safety/data_analyzer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,337 @@
import os
import json
import numpy as np
from collections import defaultdict


class DataAnalyzer:
"""数据分析器 - 生成数据集统计信息"""

@staticmethod
def analyze_dataset(data_dir):
"""分析数据集并生成详细报告"""
print(f"分析数据集: {data_dir}")

analysis = {
'basic_stats': DataAnalyzer._get_basic_stats(data_dir),
'file_distribution': DataAnalyzer._analyze_file_distribution(data_dir),
'object_statistics': DataAnalyzer._analyze_objects(data_dir),
'temporal_analysis': DataAnalyzer._analyze_temporal(data_dir),
'quality_metrics': DataAnalyzer._calculate_quality_metrics(data_dir)
}

# 生成评分
analysis['overall_score'] = DataAnalyzer._calculate_overall_score(analysis)

# 保存分析结果
DataAnalyzer._save_analysis_report(data_dir, analysis)

# 打印摘要
DataAnalyzer._print_analysis_summary(analysis)

return analysis

@staticmethod
def _get_basic_stats(data_dir):
"""获取基本统计信息"""
stats = {
'total_size_mb': DataAnalyzer._get_directory_size(data_dir),
'file_count': 0,
'directory_count': 0,
'data_types': defaultdict(int)
}

for root, dirs, files in os.walk(data_dir):
stats['directory_count'] += len(dirs)
stats['file_count'] += len(files)

for file in files:
ext = os.path.splitext(file)[1].lower()
if ext in ['.png', '.jpg', '.jpeg']:
stats['data_types']['images'] += 1
elif ext == '.json':
stats['data_types']['json'] += 1
elif ext == '.txt':
stats['data_types']['text'] += 1
elif ext == '.bin':
stats['data_types']['binary'] += 1
else:
stats['data_types']['other'] += 1

return stats

@staticmethod
def _get_directory_size(path):
"""计算目录大小"""
total = 0
for dirpath, dirnames, filenames in os.walk(path):
for f in filenames:
fp = os.path.join(dirpath, f)
if os.path.exists(fp):
total += os.path.getsize(fp)
return round(total / (1024 * 1024), 2)

@staticmethod
def _analyze_file_distribution(data_dir):
"""分析文件分布"""
distribution = {}

# 分析原始图像
raw_dirs = ['vehicle', 'infrastructure']
for raw_dir in raw_dirs:
path = os.path.join(data_dir, "raw", raw_dir)
if os.path.exists(path):
camera_stats = {}
for camera_dir in os.listdir(path):
camera_path = os.path.join(path, camera_dir)
if os.path.isdir(camera_path):
images = [f for f in os.listdir(camera_path)
if f.endswith(('.png', '.jpg', '.jpeg'))]
camera_stats[camera_dir] = len(images)
distribution[f'raw_{raw_dir}'] = camera_stats

# 分析拼接图像
stitched_dir = os.path.join(data_dir, "stitched")
if os.path.exists(stitched_dir):
stitched_images = [f for f in os.listdir(stitched_dir)
if f.endswith(('.jpg', '.jpeg', '.png'))]
distribution['stitched'] = len(stitched_images)

# 分析标注文件
annotations_dir = os.path.join(data_dir, "annotations")
if os.path.exists(annotations_dir):
json_files = [f for f in os.listdir(annotations_dir) if f.endswith('.json')]
distribution['annotations'] = len(json_files)

return distribution

@staticmethod
def _analyze_objects(data_dir):
"""分析物体统计"""
annotations_dir = os.path.join(data_dir, "annotations")

if not os.path.exists(annotations_dir):
return {'total_objects': 0, 'by_class': {}, 'by_frame': {}}

object_stats = {
'total_objects': 0,
'by_class': defaultdict(int),
'by_frame': defaultdict(int),
'class_distribution': {}
}

json_files = [f for f in os.listdir(annotations_dir)
if f.endswith('.json') and f.startswith('frame_')]

for json_file in json_files:
try:
with open(os.path.join(annotations_dir, json_file), 'r') as f:
data = json.load(f)

frame_id = data.get('frame_id', 0)
objects = data.get('objects', [])

object_stats['by_frame'][frame_id] = len(objects)
object_stats['total_objects'] += len(objects)

for obj in objects:
obj_class = obj.get('class', 'unknown')
object_stats['by_class'][obj_class] += 1

except Exception as e:
print(f"分析标注文件 {json_file} 失败: {e}")

# 计算类分布百分比
if object_stats['total_objects'] > 0:
for obj_class, count in object_stats['by_class'].items():
object_stats['class_distribution'][obj_class] = round(
count / object_stats['total_objects'] * 100, 2)

return object_stats

@staticmethod
def _analyze_temporal(data_dir):
"""分析时间分布"""
temporal_stats = {
'frame_intervals': [],
'total_duration': 0,
'frame_rate': 0
}

metadata_file = os.path.join(data_dir, "metadata", "collection_info.json")
if os.path.exists(metadata_file):
try:
with open(metadata_file, 'r') as f:
metadata = json.load(f)

collection_stats = metadata.get('collection_stats', {})
temporal_stats['total_duration'] = collection_stats.get('duration_seconds', 0)
temporal_stats['frame_rate'] = collection_stats.get('average_fps', 0)

except Exception as e:
print(f"分析元数据失败: {e}")

return temporal_stats

@staticmethod
def _calculate_quality_metrics(data_dir):
"""计算质量指标"""
quality_metrics = {
'completeness_score': 0,
'consistency_score': 0,
'diversity_score': 0,
'issues_found': []
}

# 检查完整性
required_dirs = ["raw/vehicle", "raw/infrastructure", "stitched", "metadata"]
missing_dirs = []

for dir_path in required_dirs:
if not os.path.exists(os.path.join(data_dir, dir_path)):
missing_dirs.append(dir_path)

if missing_dirs:
quality_metrics['issues_found'].append(f"缺失目录: {missing_dirs}")
quality_metrics['completeness_score'] = 50
else:
quality_metrics['completeness_score'] = 100

# 检查一致性(图像数量)
raw_vehicle = os.path.join(data_dir, "raw", "vehicle")
if os.path.exists(raw_vehicle):
camera_counts = []
for camera_dir in os.listdir(raw_vehicle):
camera_path = os.path.join(raw_vehicle, camera_dir)
if os.path.isdir(camera_path):
images = [f for f in os.listdir(camera_path)
if f.endswith('.png')]
camera_counts.append(len(images))

if camera_counts:
max_diff = max(camera_counts) - min(camera_counts)
if max_diff > 5:
quality_metrics['issues_found'].append(f"摄像头图像数量不一致: 差异{max_diff}")
quality_metrics['consistency_score'] = 70
else:
quality_metrics['consistency_score'] = 95

# 多样性评分(基于物体类别)
object_stats = DataAnalyzer._analyze_objects(data_dir)
num_classes = len(object_stats.get('by_class', {}))

if num_classes >= 5:
quality_metrics['diversity_score'] = 90
elif num_classes >= 3:
quality_metrics['diversity_score'] = 70
else:
quality_metrics['diversity_score'] = 50
quality_metrics['issues_found'].append(f"物体类别较少: {num_classes}类")

return quality_metrics

@staticmethod
def _calculate_overall_score(analysis):
"""计算总体评分"""
weights = {
'completeness': 0.3,
'consistency': 0.25,
'diversity': 0.25,
'temporal': 0.2
}

quality = analysis['quality_metrics']

score = (
quality['completeness_score'] * weights['completeness'] +
quality['consistency_score'] * weights['consistency'] +
quality['diversity_score'] * weights['diversity']
)

# 时间因素
temporal = analysis['temporal_analysis']
if temporal['frame_rate'] >= 2.0:
score += 20 * weights['temporal']
elif temporal['frame_rate'] >= 1.0:
score += 15 * weights['temporal']
else:
score += 5 * weights['temporal']

return round(score, 1)

@staticmethod
def _save_analysis_report(data_dir, analysis):
"""保存分析报告"""
report_file = os.path.join(data_dir, "metadata", "dataset_analysis.json")

with open(report_file, 'w', encoding='utf-8') as f:
json.dump(analysis, f, indent=2, ensure_ascii=False)

print(f"数据集分析报告保存: {report_file}")

@staticmethod
def _print_analysis_summary(analysis):
"""打印分析摘要"""
print("\n" + "=" * 60)
print("数据集分析摘要")
print("=" * 60)

basic = analysis['basic_stats']
print(f"\n基本统计:")
print(f" 总大小: {basic['total_size_mb']} MB")
print(f" 文件数: {basic['file_count']}")
print(f" 目录数: {basic['directory_count']}")

print(f" 数据类型:")
for data_type, count in basic['data_types'].items():
print(f" {data_type}: {count}")

distribution = analysis['file_distribution']
print(f"\n文件分布:")
for key, value in distribution.items():
if isinstance(value, dict):
print(f" {key}:")
for subkey, subvalue in value.items():
print(f" {subkey}: {subvalue}")
else:
print(f" {key}: {value}")

objects = analysis['object_statistics']
print(f"\n物体统计:")
print(f" 总物体数: {objects['total_objects']}")
if objects['by_class']:
print(f" 类别分布:")
for obj_class, count in objects['by_class'].items():
percentage = objects['class_distribution'].get(obj_class, 0)
print(f" {obj_class}: {count} ({percentage}%)")

temporal = analysis['temporal_analysis']
print(f"\n时间分析:")
print(f" 总时长: {temporal['total_duration']:.1f}秒")
print(f" 平均帧率: {temporal['frame_rate']:.2f} FPS")

quality = analysis['quality_metrics']
print(f"\n质量指标:")
print(f" 完整性: {quality['completeness_score']}/100")
print(f" 一致性: {quality['consistency_score']}/100")
print(f" 多样性: {quality['diversity_score']}/100")

if quality['issues_found']:
print(f" 发现的问题 ({len(quality['issues_found'])}):")
for issue in quality['issues_found'][:3]:
print(f" - {issue}")
if len(quality['issues_found']) > 3:
print(f" ... 还有 {len(quality['issues_found']) - 3} 个问题")

print(f"\n总体评分: {analysis['overall_score']}/100")

if analysis['overall_score'] >= 90:
print("✓ 数据集质量优秀")
elif analysis['overall_score'] >= 75:
print("✓ 数据集质量良好")
elif analysis['overall_score'] >= 60:
print("⚠ 数据集质量一般")
else:
print("✗ 数据集质量需要改进")

print("=" * 60)
Loading
Loading