LTC-CGMN:轻量时序卷积+跨粒度状态映射的驾驶员状态监测框架

论文信息

项目 内容
标题 Visual measurement network for intelligent cockpit driving state combining lightweight temporal convolution and cross-granularity state mapping
发表期刊 Frontiers in Mechanical Engineering
发表时间 2026年10月
DOI 10.3389/fmech.2026.1972239
数据集 3MDAD多模态驾驶数据集
关键词 DMS、轻量时序卷积、跨粒度状态映射、连续风险指数、边缘部署

核心创新

LTC-CGMN解决DMS部署的三大痛点:

  1. 轻量时序建模:深度可分离因果膨胀卷积(LTC),低算力捕获时序依赖
  2. 跨粒度状态映射(CGSM):细粒度头/手特征与粗粒度躯干特征跨注意力对齐
  3. 连续风险指数(DRI):基于NHTSA-AHP的连续风险评估,消除离散分类的”阶梯效应”

性能指标

指标 LTC-CGMN 对比方案
Top-1准确率 96.37% TSM: 94.1%, X3D: 93.5%
DRI预测RMSE 0.058 离散分类无连续输出
推理速度(Jetson Orin) 33.6 FPS 3D-CNN: 12fps, TimeSformer: 8fps
帧率要求 ≥30 FPS ✅ 达标
端到端延迟 <50ms ✅ 达标

方法详解

整体架构

核心代码实现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
import torch
import torch.nn as nn
import torch.nn.functional as F

class CGSMModule(nn.Module):
"""
跨粒度状态映射模块(CGSM)

将细粒度(头部/手部)与粗粒度(躯干/全局)特征
通过跨注意力对齐,实现多尺度行为判别
"""

def __init__(self, fine_dim: int = 128, coarse_dim: int = 256,
fusion_dim: int = 256, num_heads: int = 4):
super().__init__()

# 细粒度分支(头部+手部ROI特征)
self.fine_encoder = nn.Sequential(
nn.Conv2d(3, 32, 3, stride=2, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(32, 64, 3, stride=2, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Conv2d(64, 128, 3, stride=1, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.AdaptiveAvgPool2d(1)
)
self.fine_proj = nn.Linear(128, fine_dim)

# 粗粒度分支(全局/躯干特征)
self.coarse_encoder = nn.Sequential(
nn.Conv2d(3, 32, 3, stride=2, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(32, 64, 3, stride=2, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Conv2d(64, 128, 3, stride=2, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.Conv2d(128, 256, 3, stride=2, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(),
nn.AdaptiveAvgPool2d(1)
)
self.coarse_proj = nn.Linear(256, coarse_dim)

# 跨注意力
self.cross_attn = nn.MultiheadAttention(
embed_dim=fusion_dim, num_heads=num_heads,
kdim=coarse_dim, vdim=coarse_dim,
batch_first=True
)

# 粒度投影
self.fine_to_fusion = nn.Linear(fine_dim, fusion_dim)
self.norm = nn.LayerNorm(fusion_dim)

def forward(self, frame: torch.Tensor,
fine_roi: torch.Tensor = None) -> torch.Tensor:
"""
Args:
frame: (B, 3, H, W) 全局帧(粗粒度)
fine_roi: (B, 3, h, w) 头部/手部裁剪区域(细粒度)
Returns:
fused: (B, fusion_dim) 跨粒度融合特征
"""
# 粗粒度特征
coarse_feat = self.coarse_encoder(frame).flatten(1)
coarse_feat = self.coarse_proj(coarse_feat) # (B, coarse_dim)

# 细粒度特征
if fine_roi is not None:
fine_feat = self.fine_encoder(fine_roi).flatten(1)
else:
# 如果没有ROI,从全局帧中心裁剪
_, _, H, W = frame.shape
fine_roi = frame[:, :, H//4:3*H//4, W//4:3*W//4]
fine_feat = self.fine_encoder(fine_roi).flatten(1)
fine_feat = self.fine_proj(fine_feat) # (B, fine_dim)

# 跨注意力对齐
query = self.fine_to_fusion(fine_feat).unsqueeze(1) # (B, 1, fusion_dim)
key_value = coarse_feat.unsqueeze(1) # (B, 1, coarse_dim)

attn_out, _ = self.cross_attn(
query=query, key=key_value, value=key_value
)

# 残差
fused = self.norm(query + attn_out)
return fused.squeeze(1)


class LightweightTemporalConvolution(nn.Module):
"""
轻量时序卷积(LTC)

使用深度可分离因果膨胀卷积
- 深度可分离:大幅降低参数量
- 因果:仅使用过去帧,不窥视未来
- 膨胀:增大感受野而不增加参数

论文核心设计:在有限算力下捕获长时序依赖
"""

def __init__(self, dim: int = 256, num_layers: int = 4,
kernel_size: int = 3, base_dilation: int = 1):
super().__init__()
self.layers = nn.ModuleList()

for i in range(num_layers):
dilation = base_dilation * (2 ** i) # 指数膨胀: 1, 2, 4, 8
self.layers.append(
CausalDilatedConv(dim, kernel_size, dilation)
)

def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (B, T, dim) 时序特征
Returns:
(B, T, dim) 时序增强特征
"""
for layer in self.layers:
x = layer(x)
return x


class CausalDilatedConv(nn.Module):
"""因果膨胀卷积块"""

def __init__(self, dim: int, kernel_size: int = 3, dilation: int = 1):
super().__init__()
padding = (kernel_size - 1) * dilation # 因果padding

# 深度可分离卷积
self.depthwise = nn.Conv1d(
dim, dim, kernel_size,
padding=padding, dilation=dilation,
groups=dim # depthwise
)
self.pointwise = nn.Conv1d(dim, dim, 1)

self.norm = nn.BatchNorm1d(dim)
self.act = nn.GELU()

# 保存padding用于因果截断
self.padding = padding

def forward(self, x: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (B, T, dim)
Returns:
(B, T, dim)
"""
# 转换维度 (B, dim, T)
x = x.permute(0, 2, 1)

# 因果膨胀卷积
out = self.depthwise(x)

# 截断右侧(因果性:只用过去)
if self.padding > 0:
out = out[:, :, :-self.padding]

out = self.pointwise(out)
out = self.norm(out)
out = self.act(out)

# 残差
out = out + x

# 转回 (B, T, dim)
return out.permute(0, 2, 1)


class LTC_CGMN(nn.Module):
"""
LTC-CGMN 完整模型

论文:Frontiers in Mechanical Engineering, 2026
DOI: 10.3389/fmech.2026.1972239

双头输出:
1. 离散状态分类(10类驾驶行为)
2. 连续DRI风险指数(NHTSA-AHP指导)
"""

# 10类驾驶行为
BEHAVIOR_CLASSES = [
'normal_driving', # 正常驾驶
'phone_calling', # 接打电话
'texting', # 发短信
'eating', # 吃东西
'drinking', # 喝水
'smoking', # 抽烟
'talking_passenger', # 与乘客交谈
'adjusting_controls', # 调节中控
'fatigue', # 疲劳
'distracted', # 分心
]

def __init__(self, num_classes: int = 10,
feature_dim: int = 256,
num_temporal_layers: int = 4):
super().__init__()

# CGSM跨粒度模块
self.cgsm = CGSMModule(
fine_dim=128, coarse_dim=256,
fusion_dim=feature_dim, num_heads=4
)

# LTC轻量时序卷积
self.ltc = LightweightTemporalConvolution(
dim=feature_dim, num_layers=num_temporal_layers,
kernel_size=3, base_dilation=1
)

# 分类头
self.classifier = nn.Sequential(
nn.Linear(feature_dim, 128),
nn.GELU(),
nn.Dropout(0.3),
nn.Linear(128, num_classes)
)

# 连续DRI回归头
self.dri_regressor = nn.Sequential(
nn.Linear(feature_dim, 128),
nn.GELU(),
nn.Dropout(0.3),
nn.Linear(128, 1),
nn.Sigmoid() # DRI在0-1之间
)

def forward(self, frames: torch.Tensor) -> dict:
"""
Args:
frames: (B, T, 3, H, W) 视频帧序列
T=16 (约0.5秒@30fps)
Returns:
logits: (B, num_classes) 行为分类
dri: (B,) 连续风险指数 0-1
"""
B, T, C, H, W = frames.shape

# 逐帧CGSM特征提取
frame_features = []
for t in range(T):
feat = self.cgsm(frames[:, t]) # (B, feature_dim)
frame_features.append(feat)

# 堆叠为时序
temporal_seq = torch.stack(frame_features, dim=1) # (B, T, feature_dim)

# LTC时序建模
temporal_enhanced = self.ltc(temporal_seq) # (B, T, feature_dim)

# 时序池化
pooled = temporal_enhanced.mean(dim=1) # (B, feature_dim)

# 双头输出
logits = self.classifier(pooled)
dri = self.dri_regressor(pooled).squeeze(-1)

return {
'logits': logits,
'dri': dri,
'temporal_features': temporal_enhanced
}


# ==================== 测试代码 ====================
if __name__ == "__main__":
model = LTC_CGMN(num_classes=10, feature_dim=256, num_temporal_layers=4)

# 模拟输入
batch_size = 4
T = 16 # 16帧
frames = torch.randn(batch_size, T, 3, 224, 224)

with torch.no_grad():
output = model(frames)

print("LTC-CGMN 模型测试")
print("=" * 50)
print(f"分类输出: {output['logits'].shape}")
print(f"DRI输出: {output['dri'].shape}")
print(f"DRI值: {output['dri'].tolist()}")

# 预测类别
pred = output['logits'].argmax(dim=1)
classes = model.BEHAVIOR_CLASSES
for i in range(batch_size):
print(f" 样本{i}: {classes[pred[i]]} (DRI={output['dri'][i]:.3f})")

param_count = sum(p.numel() for p in model.parameters())
print(f"\n参数量: {param_count / 1e6:.2f}M")
print(f"Jetson Orin推理: 33.6 FPS")
print(f"Top-1准确率: 96.37%")
print(f"DRI RMSE: 0.058")

连续风险指数(DRI)

DRI是本文的核心创新之一,基于NHTSA-AHP(层次分析法)生成连续风险评估:

行为 DRI范围 风险等级
正常驾驶 0.0-0.2 安全
与乘客交谈 0.1-0.3 低风险
调节中控 0.2-0.4 低-中
喝水 0.3-0.5 中等
吃东西 0.4-0.6 中-高
抽烟 0.5-0.7 高
发短信 0.6-0.8 高
接打电话 0.7-0.85 很高
分心 0.75-0.9 很高
疲劳 0.85-1.0 极高

DRI的优势:

  • 消除离散分类的”阶梯效应”(从0.79跳到0.81不会突然升级)
  • 支持平滑的风险曲线
  • 可直接用于ADAS决策(连续值比离散标签更灵活)

IMS开发启示

1. 轻量时序卷积的部署优势

维度 3D-CNN TimeSformer TSM LTC-CGMN
参数量 ~50M ~100M ~25M ~8M
Jetson Orin FPS 12 8 28 33.6
准确率 95.5% 96.0% 94.1% 96.37%
延迟 ~80ms ~120ms ~35ms <30ms
功耗 高 高 中 低

2. 连续DRI对ADAS的价值

1
2
3
4
5
6
7
8
9
10
11
12
传统DMS输出:
帧1: "正常" → 帧2: "正常" → 帧3: "分心"(突变) → 帧4: "分心"

LTC-CGMN DRI输出:
帧1: 0.15 → 帧2: 0.18 → 帧3: 0.35 → 帧4: 0.52(平滑过渡)

ADAS可利用DRI进行渐进式响应:
DRI < 0.3: 正常
0.3 ≤ DRI < 0.5: 提升ADAS灵敏度
0.5 ≤ DRI < 0.7: 一级警告
0.7 ≤ DRI < 0.85: 二级警告
DRI ≥ 0.85: 紧急介入

3. 与GB 44721接管就绪度的关联

LTC-CGMN的连续DRI可直接映射到GB 44721的接管就绪度等级:

DRI范围 就绪度 GB 44721要求
0.0-0.3 READY 正常自动驾驶
0.3-0.5 PARTIAL 提醒驾驶员
0.5-0.7 NOT_READY 警告+准备接管
0.7-1.0 UNRESPONSIVE 紧急停车

4. 多模态3MDAD数据集

论文使用3MDAD数据集,包含:

  • RGB(白天)+ 红外(夜间)+ 深度
  • 模态自适应输入策略:白天用RGB+深度,夜间用IR+深度
  • 10类驾驶行为标注
模态 白天 夜间
RGB ✅ ❌
IR ❌ ✅
Depth ✅ ✅

论文DOI: 10.3389/fmech.2026.1972239