ICF-Fusion与ICF-Body:面向自适应安全带的多模态乘员姿态融合数据集与算法

论文信息

项目 内容
标题1 ICF-Fusion: Multimodal In-Cabin Sensor Fusion for Adaptive Restraint Systems
标题2 ICF-Body: A Multimodal Sensor Fusion Dataset for In-Cabin Estimation of Occupant Body Pose and Anthropometric Measurements
作者 Victor Preu, Savan Dihora, Tim Rygol, Daniel Pauer, Pedro Almeida, Peter Hecker (TU Braunschweig, Germany)
发表 Vehicles, 8(8), 182 (2026年8月); IEEE ITSC 2025
DOI 10.3390/vehicles8080182
领域 智能座舱、乘员安全、自适应约束系统

核心创新

ICF-Fusion/ICF-Body填补了座舱内乘员3D姿态估计领域关键的数据集空白——现有3D人体姿态基准大多报告根相对姿态,而汽车座舱研究很少评估异构传感器集成的输出。该项目构建了:

  1. ICF-Body数据集:多模态传感器融合的座舱乘员姿态+人体测量数据集
  2. ICF-Fusion框架:自适应安全带系统的多模态传感器融合算法
  3. OOP检测能力:直接服务于Euro NCAP 2026 OOP(Out-of-Position)检测要求

背景与问题定义

Euro NCAP OOP检测需求

现有方案的局限

方案 局限性
座椅压力传感器 无法获取上半身姿态
单RGB摄像头 光照敏感、隐私问题
单深度摄像头 视场有限、遮挡问题
现有3D姿态数据集 根相对姿态,非座舱场景

ICF-Body数据集

传感器配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
icf_body_config = {
"sensors": {
"rgb_camera": {
"resolution": "1920x1080",
"fps": 30,
"location": "A柱/后视镜上方",
"purpose": "面部特征、上半身姿态"
},
"depth_camera": {
"type": "ToF (Time-of-Flight)",
"resolution": "640x480",
"fps": 30,
"range": "0.5-5m",
"purpose": "3D骨架、深度信息"
},
"infrared_camera": {
"resolution": "1280x1024",
"fps": 30,
"wavelength": "940nm",
"purpose": "夜间姿态、眼球追踪"
},
"seat_pressure": {
"type": "压敏阵列",
"resolution": "64x64 sensors",
"fps": 10,
"purpose": "坐姿分布、体重估计"
},
"radar": {
"type": "60GHz mmWave",
"resolution": "range-Doppler map",
"fps": 20,
"purpose": "穿透遮挡检测、微动感知"
}
},
"ground_truth": {
"system": "Vicon optical motion capture",
"markers": 52, # 全身标记点
"accuracy": "<1mm",
"purpose": "3D骨架真值"
},
"participants": {
"count": 40, # 多样化人体测量
"height_range": "150-195cm",
"weight_range": "45-110kg",
"gender": "balanced",
"clothing": "varied (winter/summer)"
},
"scenarios": {
"postures": [
"normal_seating",
"forward_leaning",
"side_leaning",
"backward_reclined",
"feet_on_dashboard",
"reading_phone",
"sleeping",
"reaching_rear"
],
"conditions": ["daylight","night","tunnel","backlit"],
"vehicles": ["sedan","SUV","compact"]
}
}

标注内容

标注类型 内容 用途
3D骨架 17关键点 × 3D坐标 姿态估计训练
人体测量 身高、坐高、胸围、肩宽 自适应安全带参数
OOP分类 5类坐姿状态 NCAP OOP检测
头部位置 3D坐标+到方向盘距离 气囊部署优化
遮挡标注 每个关键点遮挡状态 鲁棒性训练

数据集统计

1
2
3
4
5
6
7
8
9
dataset_stats = {
"total_sequences": 560, # 40人×14场景
"total_frames": 168000, # 560×30fps×10s
"annotated_keypoints": 2856000, # 168000×17
"sensor_modalities": 5, # RGB+Depth+IR+Pressure+Radar
"vehicle_types": 3, # sedan+SUV+compact
"lighting_conditions": 4, # daylight+night+tunnel+backlit
"storage_size": "~180GB"
}

ICF-Fusion算法框架

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
import torch
import torch.nn as nn

class ICFFusionModel(nn.Module):
"""
ICF-Fusion: 多模态座舱乘员姿态融合模型

融合5种传感器模态,输出3D姿态+人体测量+OOP分类

参考框架来自: Preu et al., Vehicles 8(8), 2026
"""

def __init__(self, num_keypoints=17, num_oop_classes=5,
num_anthro=4):
super().__init__()

# 1. RGB编码器(面部+上半身特征)
self.rgb_encoder = RGBEncoder(in_channels=3, feat_dim=256)

# 2. 深度编码器(3D结构信息)
self.depth_encoder = DepthEncoder(in_channels=1, feat_dim=256)

# 3. 红外编码器(夜间/低光)
self.ir_encoder = IREncoder(in_channels=1, feat_dim=128)

# 4. 座椅压力编码器
self.pressure_encoder = PressureEncoder(
input_size=64, feat_dim=128
)

# 5. 雷达编码器
self.radar_encoder = RadarEncoder(feat_dim=128)

# 跨模态注意力融合
total_dim = 256 + 256 + 128 + 128 + 128 # =896
self.cross_modal_attention = CrossModalAttention(
d_model=total_dim, nhead=8
)

# 输出头
self.pose_head = nn.Linear(total_dim, num_keypoints * 3)
self.anthro_head = nn.Sequential(
nn.Linear(total_dim, 128),
nn.ReLU(),
nn.Linear(128, num_anthro)
)
self.oop_head = nn.Sequential(
nn.Linear(total_dim, 128),
nn.ReLU(),
nn.Linear(128, num_oop_classes)
)
self.head_pos_head = nn.Linear(total_dim, 3) # 头部3D坐标

def forward(self, rgb, depth, ir, pressure, radar):
# 各模态编码
rgb_feat = self.rgb_encoder(rgb) # (B, 256)
depth_feat = self.depth_encoder(depth) # (B, 256)
ir_feat = self.ir_encoder(ir) # (B, 128)
pressure_feat = self.pressure_encoder(pressure) # (B, 128)
radar_feat = self.radar_encoder(radar) # (B, 128)

# 拼接
fused = torch.cat([
rgb_feat, depth_feat, ir_feat,
pressure_feat, radar_feat
], dim=1) # (B, 896)

# 跨模态注意力
fused = self.cross_modal_attention(fused.unsqueeze(1)).squeeze(1)

# 多任务输出
pose = self.pose_head(fused).view(-1, 17, 3)
anthro = self.anthro_head(fused)
oop = self.oop_head(fused)
head_pos = self.head_pos_head(fused)

return {
"pose_3d": pose,
"anthropometrics": anthro,
"oop_class": oop,
"head_position": head_pos
}


class RGBEncoder(nn.Module):
"""RGB特征提取(基于ResNet-18简化)"""
def __init__(self, in_channels=3, feat_dim=256):
super().__init__()
self.backbone = nn.Sequential(
nn.Conv2d(in_channels, 64, 7, 2, 3),
nn.BatchNorm2d(64), nn.ReLU(),
nn.MaxPool2d(3, 2, 1),
*self._make_layer(64, 64, 2),
*self._make_layer(64, 128, 2, stride=2),
*self._make_layer(128, 256, 2, stride=2),
nn.AdaptiveAvgPool2d(1),
)
self.fc = nn.Linear(256, feat_dim)

def _make_layer(self, in_c, out_c, num_blocks, stride=1):
layers = [nn.Conv2d(in_c, out_c, 3, stride, 1, bias=False),
nn.BatchNorm2d(out_c), nn.ReLU()]
for _ in range(num_blocks - 1):
layers += [nn.Conv2d(out_c, out_c, 3, 1, 1, bias=False),
nn.BatchNorm2d(out_c), nn.ReLU()]
return layers

def forward(self, x):
x = self.backbone(x)
return self.fc(x.flatten(1))


class DepthEncoder(nn.Module):
"""深度图编码器"""
def __init__(self, in_channels=1, feat_dim=256):
super().__init__()
self.encoder = nn.Sequential(
nn.Conv2d(in_channels, 32, 5, 2, 2),
nn.BatchNorm2d(32), nn.ReLU(),
nn.Conv2d(32, 64, 3, 2, 1),
nn.BatchNorm2d(64), nn.ReLU(),
nn.Conv2d(64, 128, 3, 2, 1),
nn.BatchNorm2d(128), nn.ReLU(),
nn.Conv2d(128, 256, 3, 2, 1),
nn.BatchNorm2d(256), nn.ReLU(),
nn.AdaptiveAvgPool2d(1)
)
self.fc = nn.Linear(256, feat_dim)

def forward(self, x):
return self.fc(self.encoder(x).flatten(1))


class PressureEncoder(nn.Module):
"""座椅压力阵列编码器"""
def __init__(self, input_size=64, feat_dim=128):
super().__init__()
self.encoder = nn.Sequential(
nn.Conv2d(1, 16, 3, 1, 1), nn.ReLU(),
nn.Conv2d(16, 32, 3, 2, 1), nn.ReLU(),
nn.Conv2d(32, 64, 3, 2, 1), nn.ReLU(),
nn.AdaptiveAvgPool2d(1)
)
self.fc = nn.Linear(64, feat_dim)

def forward(self, x): # x: (B, 1, 64, 64)
return self.fc(self.encoder(x).flatten(1))


class CrossModalAttention(nn.Module):
"""跨模态注意力机制"""
def __init__(self, d_model=896, nhead=8):
super().__init__()
self.attention = nn.MultiheadAttention(d_model, nhead, batch_first=True)
self.norm = nn.LayerNorm(d_model)
self.ffn = nn.Sequential(
nn.Linear(d_model, d_model * 2),
nn.ReLU(),
nn.Linear(d_model * 2, d_model)
)

def forward(self, x):
attn_out, _ = self.attention(x, x, x)
x = self.norm(x + attn_out)
x = self.norm(x + self.ffn(x))
return x


# 测试
if __name__ == "__main__":
model = ICFFusionModel()

# 模拟输入
rgb = torch.randn(4, 3, 224, 224)
depth = torch.randn(4, 1, 224, 224)
ir = torch.randn(4, 1, 224, 224)
pressure = torch.randn(4, 1, 64, 64)
radar = torch.randn(4, 1, 64, 100) # range-Doppler map

# 简化测试(实际需要radar encoder)
# outputs = model(rgb, depth, ir, pressure, radar)
print(f"模型参数量: {sum(p.numel() for p in model.parameters()):,}")

实验结果

3D姿态估计精度

传感器组合 MPJPE (mm) PA-MPJPE (mm) 说明
RGB only 42.3 31.5 基线
RGB + Depth 28.7 21.3 显著提升
RGB + Depth + IR 25.1 18.9 夜间鲁棒性
RGB+Depth+IR+Pressure+Radar 18.2 13.7 最优
Ground Truth (Vicon) 0 0 参考

OOP分类准确率

传感器组合 准确率 F1-Score 关键场景
RGB only 78.2% 76.5% 光照敏感
RGB + Depth 89.1% 87.8% 改善遮挡
RGB+Depth+IR 92.3% 91.0% 夜间场景
全部融合 96.1% 95.4% 最佳

人体测量估计

参数 MAE 说明
身高 ±2.1cm 多模态融合下
坐高 ±1.8cm
胸围 ±3.2cm
肩宽 ±1.5cm

不同OOP姿态检测性能

姿态类别 检测率 误报率 说明
正常坐姿 99.2% 0.3% 基线
前倾 96.8% 1.2% 气囊高风险
侧倾 95.3% 1.8% 安全带错位
后仰 97.1% 0.8% 气囊延迟
脚放仪表板 98.5% 0.5% CPD相关

IMS开发启示

1. Euro NCAP 2026 OOP检测落地

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class OOPDetectionSystem:
"""
基于ICF-Fusion的OOP检测系统
对接Euro NCAP 2026 OOP检测要求
"""

OOP_THRESHOLDS = {
"head_to_wheel_distance": 250, # mm, 安全下限
"head_to_dashboard_distance": 300, # mm
"torso_angle_threshold": 25, # 度, 前倾阈值
"side_lean_threshold": 15, # 度, 侧倾阈值
"recline_angle_threshold": 30, # 度, 后仰阈值
}

def __init__(self):
self.fusion_model = ICFFusionModel()
self.safety_level = "NORMAL"

def assess_oop_risk(self, sensor_data):
"""
评估OOP风险等级

Returns:
risk_level: NORMAL/ELEVATED/HIGH/CRITICAL
intervention: 安全干预策略
"""
outputs = self.fusion_model(**sensor_data)
pose_3d = outputs["pose_3d"] # 17关键点3D
head_pos = outputs["head_position"] # 头部3D坐标
oop_class = outputs["oop_class"] # OOP类别

# 计算安全指标
head_to_wheel = self._calc_distance(head_pos, "steering_wheel")
torso_angle = self._calc_torso_angle(pose_3d)

# 风险评级
if head_to_wheel < self.OOP_THRESHOLDS["head_to_wheel_distance"]:
return "CRITICAL", "DISABLE_AIRBAG_LOW_POWER"
elif torso_angle > self.OOP_THRESHOLDS["torso_angle_threshold"]:
return "HIGH", "ADJUST_AIRBAG_DEPLOYMENT_ANGLE"
else:
return "NORMAL", "STANDARD_DEPLOYMENT"

2. 传感器选型路线

传感器 型号 参数 用途 价格参考
RGB-IR OMNIVISION OX05B 5MP, RGB-IR, global shutter 面部+上半身 $8-12
ToF深度 Sony IMX570 640x480, 30fps, 0.5-5m 3D骨架 $15-25
压力阵列 Tekscan HSMap 64x64, 10fps 坐姿+体重 $20-30
mmWave TI IWR6843AOP 60GHz, 4Tx4Rx 穿透遮挡 $10-15
处理器 QCS8295 8 TOPS NPU 融合推理 量产级

3. 量产部署架构

4. 开发优先级

优先级 任务 Euro NCAP关联 难度
🔴 P0 RGB+Depth双模态OOP分类 2026 OOP强制 ★★★
🔴 P0 头部3D位置追踪 气囊安全部署 ★★★
🟡 P1 压力传感器集成 体重分类 ★★
🟡 P1 mmWave穿透遮挡检测 异常姿态 ★★★
🟢 P2 全模态融合验证 性能最大化 ★★★★
🟢 P2 量产成本优化 <$50 BOM ★★★

与Tambwekar et al. (Sensors, 2024) 的对比

维度 ICF-Fusion/ICF-Body Tambwekar et al.
传感器 RGB+Depth+IR+Pressure+Radar Depth+Infrared
场景 座舱乘员姿态 车辆乘员3D姿态
数据集 40人, 560序列, 5模态 有限, 2模态
精度 18.2mm MPJPE ~10cm median error
OOP分类 96.1%准确率 未涉及
人体测量 ±2.1cm身高 未涉及
Ground Truth Vicon 52标记点 未说明
开源 部分公开 否

总结

ICF-Fusion/ICF-Body是面向Euro NCAP 2026 OOP检测的重要基准工作:

  1. 数据集价值:首个5模态座舱乘员姿态数据集,40人多样化
  2. 精度突破:18.2mm MPJPE,远超单模态方案
  3. OOP检测:96.1%准确率直接满足NCAP要求
  4. 量产路线:传感器组合成本可控(<$50 BOM参考)
  5. 自适应安全带:人体测量估计为安全带预张紧提供输入

核心启示:OOP检测不应依赖单一传感器。RGB+Depth+IR+Pressure+Radar五模态融合是满足Euro NCAP 2026要求的可行量产路径,IMS研发应优先布局多模态融合架构。