1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
| import torch
import torch.nn as nn
from typing import Tuple
class MicroGesture3DCNN(nn.Module):
"""
3D-MobileNetV2 for Driver Micro Hand Gesture Recognition
关键设计：
- 3D深度可分离卷积（减少参数）
- 双分支（左/右手独立）
- 32帧时序窗口
"""
def __init__(self, num_classes: int = 7, input_channels: int = 3):
super().__init__()
def conv_bn(inp, oup, kernel=3, stride=1, groups=1):
return nn.Sequential(
nn.Conv3d(inp, oup, kernel, stride, kernel//2, groups=groups, bias=False),
nn.BatchNorm3d(oup),
nn.ReLU6(inplace=True),
)
def conv_dw(inp, oup, kernel=3, stride=1):
return nn.Sequential(
nn.Conv3d(inp, inp, kernel, stride, kernel//2, groups=inp, bias=False),
nn.BatchNorm3d(inp),
nn.ReLU6(inplace=True),
nn.Conv3d(inp, oup, 1, 1, 0, bias=False),
nn.BatchNorm3d(oup),
nn.ReLU6(inplace=True),
)
self.features = nn.Sequential(
conv_bn(input_channels, 32, kernel=7, stride=2),
conv_dw(32, 64, stride=2),
conv_dw(64, 128, stride=2),
conv_dw(128, 128, stride=1),
conv_dw(128, 256, stride=2),
conv_dw(256, 256, stride=1),
nn.AdaptiveAvgPool3d(1),
)
self.classifier = nn.Sequential(
nn.Dropout(0.2),
nn.Linear(256, num_classes),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (B, C, T, H, W) — T=32帧
Returns:
logits: (B, num_classes)
"""
x = self.features(x)
x = x.flatten(1)
return self.classifier(x)
class MultiModalFusion:
"""
分数级融合 (RGB + IR + Depth)
论文发现：IR > RGB > Depth（车载环境）
"""
def __init__(self, weights: dict = None):
self.weights = weights or {"rgb": 0.25, "ir": 0.50, "depth": 0.25}
def fuse(
self,
rgb_scores: torch.Tensor,
ir_scores: torch.Tensor,
depth_scores: torch.Tensor,
) -> torch.Tensor:
"""
分数级加权融合
IR权重最高（91.56%准确率）
Depth权重最低（强光下性能差）
"""
fused = (
self.weights["rgb"] * rgb_scores +
self.weights["ir"] * ir_scores +
self.weights["depth"] * depth_scores
)
return fused
class OnlineRecognition:
"""
在线识别算法
使用滑动窗口+转移概率：
- 不需要独立检测器
- 从分类器分数直接检测手势
"""
def __init__(
self,
window_size: int = 32,
stride: int = 4,
transition_threshold: float = 0.5,
):
self.window_size = window_size
self.stride = stride
self.transition_threshold = transition_threshold
def process_stream(
self,
score_stream: torch.Tensor,
) -> list:
"""
在线流式识别
Returns:
events: [{"gesture": str, "time": int, "confidence": float}]
"""
events = []
T = len(score_stream)
for t in range(0, T - self.window_size, self.stride):
window = score_stream[t:t + self.window_size]
mean_scores = window.mean(dim=0)
max_class = mean_scores.argmax().item()
max_score = mean_scores[max_class].item()
if max_class != 5 and max_score > self.transition_threshold:
if not events or events[-1]["gesture"] != max_class:
events.append({
"gesture": max_class,
"time": t,
"confidence": max_score,
})
return events
if __name__ == "__main__":
model = MicroGesture3DCNN(num_classes=7)
x = torch.randn(4, 3, 32, 120, 160)
output = model(x)
print(f"输出: {output.shape}")
print(f"参数: {sum(p.numel() for p in model.parameters()) / 1e6:.1f}M")
fusion = MultiModalFusion()
rgb = torch.softmax(torch.randn(4, 7), dim=1)
ir = torch.softmax(torch.randn(4, 7), dim=1)
depth = torch.softmax(torch.randn(4, 7), dim=1)
fused = fusion.fuse(rgb, ir, depth)
print(f"融合输出: {fused.shape}")
|