TPA-Flow:基于张量积注意力与校正流的rPPG心率监测新SOTA

论文信息

项目 内容
标题 TPA-Flow: Physiological Tensor Interactions and Rectified-Flow Decoding for Remote Photoplethysmography
来源 Research Square (预印本)
发表时间 2026年10月
关键词 rPPG、张量积注意力、校正流、心率估计、波形重建
测试集 PURE, UBFC-rPPG, MMPD

核心创新

TPA-Flow提出三大创新点解决rPPG领域的核心难题:

  1. 生理张量积注意力(TPA):将置信度加权的周期性先验注入注意力机制
  2. 直接锚定校正流解码:先产生稳定波形锚点,再用连续传输路径修正
  3. 双路径融合:保留周期稳定性 + 波形保真度

性能指标(SOTA)

数据集 MAE (BPM) RMSE (BPM) 说明
PURE 0.04 0.05 实验室级精度
UBFC-rPPG 0.47 0.62 标准基准
MMPD 4.13 7.01 跨域挑战集

与前代方法对比

方法 PURE MAE UBFC MAE MMPD MAE 跨域泛化
DeepPhys 0.87 0.78 12.5 差
PhysFormer 0.15 0.38 8.20 中
TS-CAN 0.32 0.45 9.80 中
TPA-Flow 0.04 0.47 4.13 强

方法详解

核心代码实现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
import torch
import torch.nn as nn
import torch.nn.functional as F

class TPAFlowModel(nn.Module):
"""
TPA-Flow rPPG心率估计模型

论文核心架构:
1. 质量感知生理表示
2. 张量积注意力(TPA) - 捕获面部区域×心动周期×特征子空间交互
3. 直接分支 - 稳定波形锚点
4. 流分支 - 连续传输路径修正
5. 双路径融合
"""

def __init__(self, num_regions: int = 8, num_cycles: int = 4,
feature_dim: int = 128, num_heads: int = 4):
super().__init__()

# 面部区域特征提取
self.region_encoder = FaceRegionEncoder(
num_regions=num_regions,
out_dim=feature_dim
)

# 质量评估模块
self.quality_assessor = QualityAssessor(feature_dim)

# 张量积注意力
self.tpa = PhysiologicalTensorProductAttention(
feature_dim=feature_dim,
num_regions=num_regions,
num_cycles=num_cycles,
num_heads=num_heads
)

# 直接分支(稳定锚点)
self.direct_branch = DirectDecoder(feature_dim)

# 流分支(校正传输)
self.flow_branch = RectifiedFlowDecoder(feature_dim)

# 融合
self.fusion = nn.Sequential(
nn.Linear(feature_dim * 2, feature_dim),
nn.GELU(),
nn.Linear(feature_dim, 1) # 输出rPPG波形
)

def forward(self, video: torch.Tensor) -> dict:
"""
Args:
video: (B, T, C, H, W) 面部视频
B=batch, T=时间步(180=6s@30fps)
Returns:
rppg_signal: (B, T) rPPG波形
quality: (B,) 信号质量评分
"""
B, T, C, H, W = video.shape

# 1. 面部区域特征
region_feats = self.region_encoder(video) # (B, T, num_regions, feature_dim)

# 2. 质量评估
quality = self.quality_assessor(region_feats) # (B, T)

# 3. TPA注意力
attended = self.tpa(region_feats, quality) # (B, T, feature_dim)

# 4. 直接分支:稳定锚点波形
direct_anchor = self.direct_branch(attended) # (B, T)

# 5. 流分支:校正传输
flow_corrected = self.flow_branch(attended, direct_anchor) # (B, T)

# 6. 双路径融合
fused = torch.cat([
direct_anchor.unsqueeze(-1),
flow_corrected.unsqueeze(-1)
], dim=-1) # (B, T, 2)
rppg = self.fusion(fused).squeeze(-1) # (B, T)

return {
'rppg_signal': rppg,
'quality': quality,
'direct_anchor': direct_anchor,
'flow_corrected': flow_corrected
}


class FaceRegionEncoder(nn.Module):
"""面部区域特征编码器"""

def __init__(self, num_regions: int = 8, out_dim: int = 128):
super().__init__()
self.num_regions = num_regions

# 轻量级3D CNN
self.stem = nn.Sequential(
nn.Conv3d(3, 32, (3, 3, 3), stride=(1, 2, 2), padding=(1, 1, 1)),
nn.BatchNorm3d(32),
nn.ReLU(),
)
self.layer2 = nn.Sequential(
nn.Conv3d(32, 64, (3, 3, 3), stride=(1, 2, 2), padding=(1, 1, 1)),
nn.BatchNorm3d(64),
nn.ReLU(),
)
self.layer3 = nn.Sequential(
nn.Conv3d(64, 128, (3, 3, 3), stride=(1, 1, 1), padding=(1, 1, 1)),
nn.BatchNorm3d(128),
nn.ReLU(),
)

# 区域注意力(学习面部不同区域权重)
self.region_attention = nn.MultiheadAttention(
embed_dim=out_dim, num_heads=4, batch_first=True
)

# 输出投影
self.proj = nn.Linear(128, out_dim)

def forward(self, video: torch.Tensor) -> torch.Tensor:
"""
Args:
video: (B, T, C, H, W)
Returns:
features: (B, T, num_regions, out_dim)
"""
B, T, C, H, W = video.shape

# 3D CNN提取时空特征
x = video.permute(0, 2, 1, 3, 4) # (B, C, T, H, W)
x = self.stem(x) # (B, 32, T, H/2, W/2)
x = self.layer2(x) # (B, 64, T, H/4, W/4)
x = self.layer3(x) # (B, 128, T, H/4, W/4)

# 空间分区域
_, _, T_, H_, W_ = x.shape
region_h = H_ // 2
region_w = W_ // 4
regions = []
for i in range(2):
for j in range(4):
r = x[:, :, :, i*region_h:(i+1)*region_h,
j*region_w:(j+1)*region_w]
r = F.adaptive_avg_pool3d(r, (T_, 1, 1)).squeeze(-1).squeeze(-1)
regions.append(self.proj(r.permute(0, 2, 1))) # (B, T, out_dim)

# 堆叠为区域序列
region_feats = torch.stack(regions, dim=2) # (B, T, num_regions, out_dim)

return region_feats


class QualityAssessor(nn.Module):
"""信号质量评估器"""
def __init__(self, dim: int):
super().__init__()
self.mlp = nn.Sequential(
nn.Linear(dim, dim // 2),
nn.ReLU(),
nn.Linear(dim // 2, 1),
nn.Sigmoid()
)

def forward(self, x):
return self.mlp(x.mean(dim=2)).squeeze(-1)


class PhysiologicalTensorProductAttention(nn.Module):
"""
生理张量积注意力(TPA)

捕获三维交互:
- 面部区域 × 心动周期 × 特征子空间
使用低秩张量积避免完整三维注意力的O(n³)复杂度
"""

def __init__(self, feature_dim: int, num_regions: int = 8,
num_cycles: int = 4, num_heads: int = 4,
rank: int = 16):
super().__init__()
self.num_heads = num_heads
self.rank = rank

# 周期性先验(可学习)
self.periodic_prior = nn.Parameter(
torch.randn(num_cycles, feature_dim)
)

# 质量加权
self.quality_weight = nn.Linear(1, num_heads)

# 区域注意力
self.region_attn = nn.MultiheadAttention(
embed_dim=feature_dim, num_heads=num_heads,
batch_first=True
)

# 低秩投影
self.W_q = nn.Linear(feature_dim, rank)
self.W_k = nn.Linear(feature_dim, rank)
self.W_v = nn.Linear(feature_dim, rank)

# 输出投影
self.out_proj = nn.Linear(feature_dim, feature_dim)
self.norm = nn.LayerNorm(feature_dim)

def forward(self, region_feats: torch.Tensor,
quality: torch.Tensor) -> torch.Tensor:
"""
Args:
region_feats: (B, T, num_regions, feature_dim)
quality: (B, T) 质量评分
Returns:
attended: (B, T, feature_dim)
"""
B, T, R, D = region_feats.shape

# 周期先验注入
prior = self.periodic_prior.unsqueeze(0).unsqueeze(0) # (1, 1, num_cycles, D)
prior_expanded = prior.expand(B, T, -1, -1) # (B, T, num_cycles, D)

# 质量加权
q_weight = self.quality_weight(quality.unsqueeze(-1)) # (B, T, num_heads)

# 区域间注意力
flat_regions = region_feats.view(B * T, R, D)
attn_out, _ = self.region_attn(flat_regions, flat_regions, flat_regions)
attn_out = attn_out.view(B, T, R, D)

# 低秩张量积
q = self.W_q(attn_out) # (B, T, R, rank)
k = self.W_k(prior_expanded.expand(-1, -1, R, -1)) # (B, T, R, rank)
v = self.W_v(prior_expanded.expand(-1, -1, R, -1))

# 张量积
tp_output = torch.einsum('btrd,btrd->btd', q, v) # (B, T, rank)
tp_output = self.out_proj(
F.pad(tp_output, (0, D - self.rank))
)

# 残差连接
output = self.norm(region_feats.mean(dim=2) + tp_output)

return output


class DirectDecoder(nn.Module):
"""直接解码分支:输出稳定波形锚点"""
def __init__(self, dim: int):
super().__init__()
self.decoder = nn.Sequential(
nn.Linear(dim, dim * 2),
nn.GELU(),
nn.Linear(dim * 2, dim),
nn.GELU(),
nn.Linear(dim, 1)
)

def forward(self, x):
return self.decoder(x).squeeze(-1)


class RectifiedFlowDecoder(nn.Module):
"""校正流解码分支:修正锚点波形的相位和形态"""
def __init__(self, dim: int, num_steps: int = 5):
super().__init__()
self.num_steps = num_steps

# 条件编码
self.cond_encoder = nn.Linear(dim + 1, dim) # +1 for anchor

# 流网络
self.flow_net = nn.Sequential(
nn.Linear(dim * 2, dim * 4),
nn.GELU(),
nn.Linear(dim * 4, dim * 2),
nn.GELU(),
nn.Linear(dim * 2, dim)
)

def forward(self, x: torch.Tensor, anchor: torch.Tensor) -> torch.Tensor:
"""
Args:
x: (B, T, dim) 条件特征
anchor: (B, T) 锚点波形
Returns:
corrected: (B, T) 校正后波形
"""
# 条件编码
cond = self.cond_encoder(
torch.cat([x, anchor.unsqueeze(-1)], dim=-1)
) # (B, T, dim)

# 校正流
z = anchor.unsqueeze(-1) # 初始点
for step in range(self.num_steps):
flow = self.flow_net(torch.cat([z, cond], dim=-1))
z = z + flow / self.num_steps # 线性传输

return z.squeeze(-1)


# ==================== 测试代码 ====================
if __name__ == "__main__":
model = TPAFlowModel(
num_regions=8, num_cycles=4,
feature_dim=128, num_heads=4
)

# 模拟输入
batch_size = 2
T = 180 # 6秒@30fps
video = torch.randn(batch_size, T, 3, 128, 128)

with torch.no_grad():
output = model(video)

print("TPA-Flow 模型测试")
print("=" * 50)
print(f"rPPG信号形状: {output['rppg_signal'].shape}")
print(f"质量评分形状: {output['quality'].shape}")
print(f"直接锚点形状: {output['direct_anchor'].shape}")
print(f"校正流形状: {output['flow_corrected'].shape}")

# 心率估计
import numpy as np
rppg = output['rppg_signal'][0].numpy()
fft = np.abs(np.fft.rfft(rppg))
freqs = np.fft.rfftfreq(len(rppg), 1/30)
hr_mask = (freqs >= 0.67) & (freqs <= 3.0)
hr_freq = freqs[hr_mask][np.argmax(fft[hr_mask])]
print(f"\n估计心率: {hr_freq * 60:.1f} BPM")

param_count = sum(p.numel() for p in model.parameters())
print(f"参数量: {param_count / 1e6:.2f}M")

车内rPPG应用方案

DMS摄像头→rPPG→健康监测

驾驶场景性能预估

场景 预期MAE 挑战 缓解方案
白天城市 ~1 BPM 光照变化 HDR摄像头
夜间IR ~2 BPM IR反射不均 多LED补光
隧道进出口 ~3 BPM 光照剧变 质量评估+跳帧
阳光直射 ~4 BPM 过曝/阴影 ROI自动切换
戴眼镜 ~2 BPM 眼镜反射 鼻梁/额头ROI

IMS开发启示

1. rPPG替代接触式传感器

TPA-Flow的MAE 0.04-4.13 BPM意味着rPPG已接近医疗级精度。对IMS而言:

  • 心率监测:从DMS摄像头非接触提取,无需额外传感器
  • HRV分析:RMSSD可作为疲劳指标
  • 健康预警:心律异常、心率骤变可触发告警

2. 与疲劳检测融合

1
2
3
4
5
6
7
8
9
10
11
12
13
14
IMS疲劳检测增强方案:

当前方案:
PERCLOS + 眨眼频率 + 头部姿态 → 疲劳评分

增强方案:
PERCLOS + 眨眼 + 头部姿态 + rPPG-HRV → 综合疲劳评分

权重建议(基于文献meta分析):
- PERCLOS: 30%
- 眨眼: 15%
- 头部姿态: 15%
- rPPG HRV: 25%
- 驾驶行为: 15%

3. 部署评估

平台 TPA-Flow推理时间 内存 可行性
QCS8255 ~15ms/帧 <50MB ✅ 实时
TI TDA4 ~10ms/帧 <40MB ✅ 实时
NVIDIA Orin ~5ms/帧 <80MB ✅ 实时

4. 数据需求

数据类型 数量 来源
驾驶场景rPPG 5000+视频 DMS实采
Ground truth 同步PPG 手指脉搏仪
多场景覆盖 白天/夜间/隧道 实路采集

论文来源: Research Square, rs-11099271