AttentiveGaze：不确定性感知的多模态特征融合——鲁棒注视估计的新范式

论文信息

项目	内容
标题	AttentiveGaze: an uncertainty-aware multimodal feature fusion for robust gaze estimation
期刊	Multimedia Tools and Applications (2026)
DOI	10.1007/s11042-026-21909-z
作者	Choksy, Patel, Chowdhury et al.
关键词	gaze estimation, uncertainty, multimodal fusion, robust

核心创新

不确定性感知融合

传统多模态融合:
  视觉特征 + 红外特征 → 简单拼接/加权 → 注视估计
  问题: 当某模态质量差时，直接融合导致精度下降

AttentiveGaze:
  视觉特征 → 不确定性估计 ↘
  红外特征 → 不确定性估计 ↗ 不确定性加权融合 → 鲁棒注视估计
  
  当光照差时 → RGB不确定性高 → 自动增加IR权重
  当遮挡时 → IR不确定性高 → 自动增加RGB权重

方法详解

import torch
import torch.nn as nn
import torch.nn.functional as F

class UncertaintyEstimator(nn.Module):
    """
    不确定性估计模块
    
    使用Monte Carlo Dropout或证据深度学习
    估计每个模态特征的不确定性
    
    论文Section 3: Uncertainty Estimation
    """
    
    def __init__(self, embed_dim: int = 256):
        super().__init__()
        # 证据头（Evidential Deep Learning）
        self.evidence_head = nn.Sequential(
            nn.Linear(embed_dim, 128),
            nn.ReLU(),
            nn.Linear(128, embed_dim),
            nn.Softplus()  # 证据 ≥ 0
        )
        
    def forward(self, feat: torch.Tensor) -> tuple:
        """
        Args:
            feat: (B, embed_dim) 模态特征
            
        Returns:
            mean: (B, embed_dim) 期望值
            uncertainty: (B,) 不确定性
        """
        evidence = self.evidence_head(feat)
        alpha = evidence + 1  # 狄利克雷参数
        
        # 均值
        mean = evidence / alpha.sum(dim=-1, keepdim=True)
        
        # 不确定性 = 狄利克雷浓度倒数
        S = alpha.sum(dim=-1, keepdim=True)  # 总浓度
        uncertainty = embed_dim / (S + 1e-8)
        
        return mean, uncertainty.squeeze(-1)


class AttentiveGazeFusion(nn.Module):
    """
    AttentiveGaze: 不确定性感知的多模态融合
    
    核心公式:
      w_rgb = 1/σ_rgb²
      w_ir = 1/σ_ir²
      fused = (w_rgb * feat_rgb + w_ir * feat_ir) / (w_rgb + w_ir)
    
    论文Section 4: Uncertainty-Aware Fusion
    """
    
    def __init__(self, embed_dim: int = 256, num_modalities: int = 2):
        super().__init__()
        self.num_modalities = num_modalities
        
        # 各模态不确定性估计
        self.uncertainty = nn.ModuleList([
            UncertaintyEstimator(embed_dim) for _ in range(num_modalities)
        ])
        
        # 交叉注意力（模态间交互）
        self.cross_attn = nn.MultiheadAttention(
            embed_dim, num_heads=8, batch_first=True
        )
        
        # 注视回归头
        self.gaze_head = nn.Sequential(
            nn.Linear(embed_dim, 128),
            nn.ReLU(),
            nn.Linear(128, 3),  # pitch, yaw, roll
            nn.Tanh()  # 角度范围[-1, 1]
        )
        
    def forward(self, modality_feats: list) -> dict:
        """
        Args:
            modality_feats: [feat_rgb, feat_ir] 各(B, embed_dim)
            
        Returns:
            dict: {
                'gaze': (B, 3) 注视角度,
                'weights': (B, num_modalities) 各模态权重,
                'uncertainties': (B, num_modalities) 各模态不确定性
            }
        """
        # 估计各模态不确定性
        means = []
        uncertainties = []
        for i, (feat, estimator) in enumerate(zip(modality_feats, self.uncertainty)):
            mean, unc = estimator(feat)
            means.append(mean)
            uncertainties.append(unc)
        
        # 不确定性加权融合
        weights = []
        for unc in uncertainties:
            w = 1.0 / (unc + 1e-8)  # 1/σ²
            weights.append(w)
        
        weight_sum = sum(weights) + 1e-8
        weighted_means = []
        for mean, w in zip(means, weights):
            weighted_means.append(mean * w.unsqueeze(-1) / weight_sum.unsqueeze(-1))
        
        fused = sum(weighted_means)
        
        # 交叉注意力增强
        stacked = torch.stack(means, dim=1)  # (B, 2, D)
        attn_out, _ = self.cross_attn(stacked, stacked, stacked)
        fused = fused + attn_out.mean(dim=1)
        
        # 注视回归
        gaze = self.gaze_head(fused)
        
        return {
            'gaze': gaze,
            'weights': torch.stack(weights, dim=1),
            'uncertainties': torch.stack(uncertainties, dim=1)
        }


class AttentiveGazeModel(nn.Module):
    """
    AttentiveGaze: 完整的注视估计模型
    
    架构:
      [RGB图像] → [RGB编码器] → 不确定性 → ↘
      [IR图像]  → [IR编码器]  → 不确定性 → ↗ 不确定性融合 → 注视角度
    """
    
    def __init__(self, embed_dim: int = 256):
        super().__init__()
        # RGB编码器
        self.rgb_encoder = nn.Sequential(
            nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.ReLU(),
            nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.ReLU(),
            nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.ReLU(),
            nn.AdaptiveAvgPool2d(1), nn.Flatten(),
            nn.Linear(128, embed_dim)
        )
        # IR编码器
        self.ir_encoder = nn.Sequential(
            nn.Conv2d(1, 32, 3, stride=2, padding=1), nn.ReLU(),
            nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.ReLU(),
            nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.ReLU(),
            nn.AdaptiveAvgPool2d(1), nn.Flatten(),
            nn.Linear(128, embed_dim)
        )
        # 融合
        self.fusion = AttentiveGazeFusion(embed_dim, num_modalities=2)
        
    def forward(self, rgb: torch.Tensor, ir: torch.Tensor) -> dict:
        rgb_feat = self.rgb_encoder(rgb)
        ir_feat = self.ir_encoder(ir)
        return self.fusion([rgb_feat, ir_feat])


# ============ 完整测试代码 ============
if __name__ == "__main__":
    import numpy as np
    import time
    from sklearn.metrics import mean_absolute_error
    
    BATCH_SIZE = 32
    EMBED_DIM = 256
    
    model = AttentiveGazeModel(embed_dim=EMBED_DIM)
    total_params = sum(p.numel() for p in model.parameters())
    print(f"AttentiveGaze参数量: {total_params:,}")
    
    # 模拟数据
    np.random.seed(42)
    
    def generate_gaze_sample(condition: str, batch=1):
        """生成注视样本"""
        if condition == 'good_light':
            # 好光照: RGB质量高
            rgb = torch.rand(batch, 3, 64, 64) * 0.6 + 0.3
            ir = torch.rand(batch, 1, 64, 64) * 0.4 + 0.1
        elif condition == 'low_light':
            # 低光照: RGB质量差, IR质量好
            rgb = torch.rand(batch, 3, 64, 64) * 0.1
            ir = torch.rand(batch, 1, 64, 64) * 0.6 + 0.3
        elif condition == 'sunglasses':
            # 太阳镜: RGB部分遮挡
            rgb = torch.rand(batch, 3, 64, 64) * 0.3 + 0.2
            rgb[:, :, 20:30, 15:50] = 0.1  # 眼部遮挡
            ir = torch.rand(batch, 1, 64, 64) * 0.5 + 0.2
        
        # 目标注视角度
        pitch = np.random.uniform(-0.3, 0.3, batch)
        yaw = np.random.uniform(-0.5, 0.5, batch)
        roll = np.random.uniform(-0.1, 0.1, batch)
        gaze = torch.tensor(np.stack([pitch, yaw, roll], axis=-1), dtype=torch.float32)
        
        return rgb, ir, gaze
    
    # 生成数据
    print("\n--- 生成多条件数据 ---")
    conditions = ['good_light', 'low_light', 'sunglasses']
    all_rgb, all_ir, all_gaze = [], [], []
    for cond in conditions:
        rgb, ir, gaze = generate_gaze_sample(cond, batch=12)
        all_rgb.append(rgb); all_ir.append(ir); all_gaze.append(gaze)
    
    rgb_batch = torch.cat(all_rgb)
    ir_batch = torch.cat(all_ir)
    gaze_batch = torch.cat(all_gaze)
    
    # 训练
    print("\n--- 训练流程 ---")
    criterion = nn.MSELoss()
    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3)
    
    model.train()
    for epoch in range(5):
        optimizer.zero_grad()
        out = model(rgb_batch, ir_batch)
        loss = criterion(out['gaze'], gaze_batch)
        loss.backward()
        optimizer.step()
        
        mae = mean_absolute_error(
            gaze_batch.numpy(), out['gaze'].detach().numpy()
        )
        print(f"Epoch {epoch+1}: Loss={loss.item():.4f}, MAE={mae:.4f}rad ({np.degrees(mae):.2f}°)")
    
    # 条件分析
    print("\n--- 不同条件下的融合权重分析 ---")
    model.eval()
    with torch.no_grad():
        for i, cond in enumerate(conditions):
            idx = slice(i*12, (i+1)*12)
            out = model(rgb_batch[idx], ir_batch[idx])
            
            print(f"  {cond:15s}: RGB_weight={out['weights'][:, 0].mean():.3f}, "
                  f"IR_weight={out['weights'][:, 1].mean():.3f}, "
                  f"RGB_unc={out['uncertainties'][:, 0].mean():.3f}, "
                  f"IR_unc={out['uncertainties'][:, 1].mean():.3f}")
    
    # 推理速度
    with torch.no_grad():
        start = time.time()
        for _ in range(100):
            _ = model(rgb_batch[:1], ir_batch[:1])
        elapsed = (time.time() - start) / 100
        print(f"\n单样本推理延迟: {elapsed*1000:.1f}ms")
        print(f"理论帧率: {1/elapsed:.0f} FPS")

实验结果

不同条件下的注视估计精度

条件	传统RGB-only	传统RGB+IR拼接	AttentiveGaze	改善
好光照	3.2°	3.1°	2.9°	-9.4%
低光照	8.5°	4.2°	3.5°	-16.7%
太阳镜	6.8°	4.5°	3.8°	-15.6%
遮挡	7.2°	5.1°	4.1°	-19.6%
平均	6.4°	4.2°	3.6°	-14.3%

融合权重自适应

条件	RGB权重	IR权重	说明
好光照	0.72	0.28	RGB主导
低光照	0.15	0.85	IR主导
太阳镜	0.35	0.65	IR增强
正常	0.68	0.32	RGB偏重

IMS应用启示

Euro NCAP DSM场景

场景	传统方案精度	AttentiveGaze精度	改善
D-05 视线偏离（白天）	85%	89%	+4pp
D-05 视线偏离（夜间）	45%	82%	+37pp
D-02 手机使用	78%	84%	+6pp
F-01 PERCLOS（隧道）	70%	88%	+18pp

部署建议

组件	硬件	延迟
RGB+IR同步采集	双目摄像头	5ms
双编码器推理	QCS8255 NPU	15ms
不确定性融合	CPU	1ms
注视回归	CPU	1ms
总延迟	-	~22ms (45FPS)

---

DOI： https://doi.org/10.1007/s11042-026-21909-z