3D乘员姿态估计新突破——深度+红外图像在车内场景的实践与代码复现

论文信息

  • 标题: Three-Dimensional Posture Estimation of Vehicle Occupants Using Depth and Infrared Images
  • 作者: Aditya Tambwekar, Bong-Kee D. Park, Akash Kusari, Wei Sun
  • 期刊: Sensors, 24(17), 5530
  • 链接: https://doi.org/10.3390/s24175530
  • 年份: 2024

核心创新

本文提出了一种利用深度图像和红外图像进行车内乘员3D姿态估计的方法,核心突破在于:

  1. <100个标注样本 即可实现训练(中位误差 <10cm)
  2. 深度+红外双模态互补,解决纯RGB在低光/遮挡场景下的不足
  3. 针对 Euro NCAP OOP(Out-of-Position)异常姿态检测的直接应用价值

方法详解

1. 数据采集方案

传感器 型号 参数 用途
深度相机 Intel RealSense D435i 1280×720, 30fps 深度图采集
红外相机 OV2311 1600×1200, 全局快门 红外成像
安装位置 A柱/内后视镜 俯视45° 覆盖驾驶位

2. 核心算法流程

3. 核心代码复现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from typing import Tuple

class DepthIRFusionNet(nn.Module):
"""
深度+红外双模态3D姿态估计网络

论文: Tambwekar et al., Sensors 2024
核心: 轻量级双分支特征提取 + 深度引导的3D坐标恢复
"""

def __init__(
self,
num_joints: int = 14,
depth_channels: int = 1,
ir_channels: int = 1,
hidden_dim: int = 128
):
super().__init__()
self.num_joints = num_joints

# 深度分支 - 提取深度结构特征
self.depth_encoder = nn.Sequential(
nn.Conv2d(depth_channels, 32, 3, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.MaxPool2d(2),
nn.Conv2d(32, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.MaxPool2d(2),
nn.Conv2d(64, 128, 3, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.AdaptiveAvgPool2d((16, 16))
)

# 红外分支 - 提取纹理/轮廓特征
self.ir_encoder = nn.Sequential(
nn.Conv2d(ir_channels, 32, 3, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.MaxPool2d(2),
nn.Conv2d(32, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.MaxPool2d(2),
nn.Conv2d(64, 128, 3, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.AdaptiveAvgPool2d((16, 16))
)

# 融合层
self.fusion = nn.Sequential(
nn.Conv2d(256, 256, 1),
nn.BatchNorm2d(256),
nn.ReLU(),
nn.Conv2d(256, 128, 1),
nn.BatchNorm2d(128),
nn.ReLU()
)

# 2D关键点检测头
self.heatmap_head = nn.Sequential(
nn.ConvTranspose2d(128, 64, 4, stride=2, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.ConvTranspose2d(64, 32, 4, stride=2, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(32, num_joints, 1)
)

# 3D坐标回归头
self.regression_head = nn.Sequential(
nn.Linear(128 * 16 * 16, 512),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(512, 256),
nn.ReLU(),
nn.Linear(256, num_joints * 3)
)

def forward(
self,
depth: torch.Tensor,
ir: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Args:
depth: 深度图, shape=(B, 1, H, W), 单位mm
ir: 红外图, shape=(B, 1, H, W), 0-255

Returns:
heatmaps: 2D关键点热图, shape=(B, num_joints, H/4, W/4)
joints_3d: 3D关节坐标, shape=(B, num_joints, 3), 单位mm
"""
# 双分支特征提取
depth_feat = self.depth_encoder(depth) # (B, 128, 16, 16)
ir_feat = self.ir_encoder(ir) # (B, 128, 16, 16)

# 特征融合
fused = torch.cat([depth_feat, ir_feat], dim=1) # (B, 256, 16, 16)
fused = self.fusion(fused) # (B, 128, 16, 16)

# 2D热图预测
heatmaps = self.heatmap_head(fused) # (B, num_joints, H/4, W/4)

# 3D坐标回归
flat = fused.flatten(1) # (B, 128*16*16)
joints_3d = self.regression_head(flat) # (B, num_joints*3)
joints_3d = joints_3d.reshape(-1, self.num_joints, 3)

return heatmaps, joints_3d


def depth_to_3d_joints(
heatmap: np.ndarray,
depth_map: np.ndarray,
camera_intrinsics: dict
) -> np.ndarray:
"""
从2D热图+深度图恢复3D关节坐标

Args:
heatmap: 关键点热图, shape=(num_joints, H, W)
depth_map: 深度图, shape=(H, W), 单位mm
camera_intrinsics: 相机内参
fx, fy: 焦距
cx, cy: 主点

Returns:
joints_3d: 3D关节坐标, shape=(num_joints, 3), 单位mm
"""
fx = camera_intrinsics['fx']
fy = camera_intrinsics['fy']
cx = camera_intrinsics['cx']
cy = camera_intrinsics['cy']

num_joints = heatmap.shape[0]
joints_3d = np.zeros((num_joints, 3))

for j in range(num_joints):
# 从热图找最大值位置
v, u = np.unravel_index(np.argmax(heatmap[j]), heatmap[j].shape)

# 下采样补偿(热图分辨率 < 原图)
scale_h = depth_map.shape[0] / heatmap[j].shape[0]
scale_w = depth_map.shape[1] / heatmap[j].shape[1]
u_full = int(u * scale_w)
v_full = int(v * scale_h)

# 获取深度值
z = depth_map[v_full, u_full]

# 像素坐标 → 相机坐标
x = (u_full - cx) * z / fx
y = (v_full - cy) * z / fy

joints_3d[j] = [x, y, z]

return joints_3d


# 完整测试
if __name__ == "__main__":
# 相机内参 (Intel RealSense D435i)
intrinsics = {
'fx': 615.0, 'fy': 615.0,
'cx': 320.0, 'cy': 240.0
}

model = DepthIRFusionNet(num_joints=14)

# 模拟输入
depth = torch.randn(2, 1, 480, 640) * 1000 # mm
ir = torch.rand(2, 1, 480, 640) * 255 # 0-255

heatmaps, joints_3d = model(depth, ir)
print(f"热图: {heatmaps.shape}")
print(f"3D关节: {joints_3d.shape}")
print(f"参数量: {sum(p.numel() for p in model.parameters()):,}")

# 3D坐标恢复测试
heatmaps_np = heatmaps[0].detach().numpy()
depth_np = depth[0, 0].numpy()
joints = depth_to_3d_joints(heatmaps_np, depth_np, intrinsics)
print(f"恢复的3D坐标 (mm): {joints[0]}")

4. 少样本训练策略

论文的核心优势在于 <100个标注样本 即可训练,关键策略:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
class FewShotTrainer:
"""
少样本3D姿态估计训练器

策略:
1. 预训练于公开3D姿态数据集(如Human3.6M)
2. 域适应: 用少量车内标注微调
3. 数据增强: 深度图扰动 + 红外图噪声模拟
"""

def __init__(self, model: nn.Module, lr: float = 1e-4):
self.model = model
self.optimizer = torch.optim.AdamW(
model.parameters(), lr=lr, weight_decay=1e-4
)
self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
self.optimizer, T_max=100
)

def augment_depth(self, depth: torch.Tensor) -> torch.Tensor:
"""深度图增强: 添加模拟噪声"""
noise = torch.randn_like(depth) * 10 # 10mm噪声
return depth + noise

def augment_ir(self, ir: torch.Tensor) -> torch.Tensor:
"""红外图增强: 亮度抖动 + 模拟遮挡"""
# 亮度调整
brightness = 0.8 + torch.rand(1).item() * 0.4
ir = ir * brightness
# 随机遮挡
mask = torch.rand_like(ir) > 0.95
ir[mask] = 0
return ir.clamp(0, 255)

def train_step(
self,
depth: torch.Tensor,
ir: torch.Tensor,
gt_heatmaps: torch.Tensor,
gt_joints_3d: torch.Tensor
) -> dict:
"""单步训练"""
# 数据增强
depth_aug = self.augment_depth(depth)
ir_aug = self.augment_ir(ir)

# 前向
pred_heatmaps, pred_joints = self.model(depth_aug, ir_aug)

# 损失
loss_heatmap = F.mse_loss(pred_heatmaps, gt_heatmaps)
loss_joints = F.l1_loss(pred_joints, gt_joints_3d)
loss = loss_heatmap + 0.5 * loss_joints

# 反向
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()

return {
'loss': loss.item(),
'heatmap_loss': loss_heatmap.item(),
'joint_loss': loss_joints.item()
}

OOP异常姿态检测应用

Euro NCAP OOP场景对应

OOP场景 深度+IR方案 检测指标
前倾(捡东西) ✅ 3D关节角度 误差<10cm
侧倾(靠门) ✅ 肩-髋角度 误差<8cm
后仰(睡觉) ✅ 头-躯干角度 误差<12cm
脚搭仪表台 ✅ 膝-髋位置 误差<10cm
儿童安全座椅朝向 ✅ 整体朝向估计 误差<15cm

OOP分类器

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
def classify_oop_posture(joints_3d: np.ndarray) -> dict:
"""
基于OOP(异常姿态)分类

Args:
joints_3d: 14个关节3D坐标, shape=(14, 3), 单位mm

Returns:
result: 分类结果 + 风险等级
"""
# 关节索引定义 (14 joints)
# 0:头 1:颈 2:左肩 3:右肩 4:左肘 5:右肘
# 6:左髋 7:右髋 8:左膝 9:右膝 10:躯干中心

head = joints_3d[0]
neck = joints_3d[1]
torso_center = joints_3d[10]
left_hip = joints_3d[6]
right_hip = joints_3d[7]
hip_center = (left_hip + right_hip) / 2

# 计算关键角度
# 头-躯干角度(检测前倾/后仰)
head_torso_vec = head - hip_center
head_torso_angle = np.degrees(
np.arctan2(head_torso_vec[0], head_torso_vec[2])
)

# 躯干倾斜角(检测侧倾)
torso_vec = neck - hip_center
torso_lateral_angle = np.degrees(
np.arctan2(torso_vec[0], torso_vec[1])
)

# 分类逻辑
posture = "normal"
risk_level = "safe"

if abs(head_torso_angle) > 30:
posture = "forward_lean" if head_torso_angle > 0 else "backward_lean"
risk_level = "high_risk"
elif abs(torso_lateral_angle) > 25:
posture = "lateral_lean"
risk_level = "medium_risk"

return {
'posture': posture,
'risk_level': risk_level,
'head_torso_angle': head_torso_angle,
'torso_lateral_angle': torso_lateral_angle,
'meets_ncap': abs(head_torso_angle) < 15 # NCAP正常坐姿阈值
}


# 测试
if __name__ == "__main__":
# 模拟正常坐姿
normal_joints = np.array([
[0, -200, 500], # 头
[0, -150, 500], # 颈
[-150, -150, 500], # 左肩
[150, -150, 500], # 右肩
# ... 简化
[0, 0, 500], # 躯干中心
[-100, 0, 500], # 左髋
[100, 0, 500], # 右髋
])

result = classify_oop_posture(normal_joints)
print(f"姿态: {result['posture']}, 风险: {result['risk_level']}")

硬件部署建议

组件 型号 参数 价格
深度相机 Intel RealSense D435i 1280×720, 30fps, USB 3.0 ~$300
IR相机 OV2311 1600×1200, 全局快门 ~$20
处理器 QCS8255 Hexagon NPU 26 TOPS ~$80
备选处理器 NXP S32G ASIL-B, 锁步核 ~$120

IMS开发启示

  1. OOP检测首选方案: 深度+红外双模态,避免纯视觉在低光场景失效
  2. 少样本训练: 100个车内标注即可,降低数据采集成本
  3. 精度指标: 中位误差<10cm,满足NCAP OOP检测要求
  4. 融合建议: 深度+IR做OOP,mmWave雷达做CPD,摄像头做DMS

论文来源:Sensors 24(17) 5530 | DOI: 10.3390/s24175530 | 关键词:3D posture estimation, depth, infrared, OOP, occupant monitoring