1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
| import torch import torch.nn as nn import torch.nn.functional as F from typing import Tuple
class MHLTDriverEmotionModel(nn.Module): """ MHLT: Multi-Head attention Level fusion for Driver emotion recognition 论文: Frontiers in Physics, 2025 核心: 视频(面部表情) + 音频(语音情绪) → 模型级融合 → 情绪强度回归 创新点: 1. 情绪连续维度 [-1, +1] 而非离散分类 2. 多头注意力模型级融合 3. 同时预测情绪类型和强度 """ def __init__(self, video_feature_dim=512, audio_feature_dim=256, hidden_dim=256, num_heads=8): super().__init__() self.video_encoder = nn.Sequential( nn.Linear(video_feature_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU(), nn.Dropout(0.1), nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU() ) self.audio_encoder = nn.Sequential( nn.Linear(audio_feature_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU(), nn.Dropout(0.1), nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU() ) self.fusion_attention = nn.MultiheadAttention( embed_dim=hidden_dim * 2, num_heads=num_heads, dropout=0.1, batch_first=True ) self.classification_head = nn.Sequential( nn.Linear(hidden_dim * 2, hidden_dim), nn.GELU(), nn.Dropout(0.2), nn.Linear(hidden_dim, 3) ) self.intensity_head = nn.Sequential( nn.Linear(hidden_dim * 2, hidden_dim), nn.GELU(), nn.Dropout(0.2), nn.Linear(hidden_dim, 1), nn.Tanh() ) def forward(self, video_features: torch.Tensor, audio_features: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: """ 前向传播 Args: video_features: 面部表情特征 [B, T, 512] audio_features: 语音情绪特征 [B, T, 256] Returns: emotion_class: 情绪分类logits [B, 3] emotion_intensity: 情绪强度 [-1, 1] [B, 1] """ v_encoded = self.video_encoder(video_features) a_encoded = self.audio_encoder(audio_features) fused = torch.cat([v_encoded, a_encoded], dim=-1) attended, _ = self.fusion_attention( fused, fused, fused ) pooled = attended.mean(dim=1) emotion_class = self.classification_head(pooled) emotion_intensity = self.intensity_head(pooled) return emotion_class, emotion_intensity
class MHLTTrainer: """MHLT模型训练器""" def __init__(self, model, lr=1e-4): self.model = model self.optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01) self.cls_criterion = nn.CrossEntropyLoss() self.reg_criterion = nn.MSELoss() def train_step(self, video_feat, audio_feat, emotion_label, intensity_label): """ 单步训练 Args: video_feat: [B, T, 512] audio_feat: [B, T, 256] emotion_label: [B] (0=正, 1=中, 2=负) intensity_label: [B] (-1.0 to 1.0) """ self.optimizer.zero_grad() cls_pred, int_pred = self.model(video_feat, audio_feat) cls_loss = self.cls_criterion(cls_pred, emotion_label) reg_loss = self.reg_criterion(int_pred.squeeze(), intensity_label) total_loss = cls_loss + 0.5 * reg_loss total_loss.backward() self.optimizer.step() return { 'total_loss': total_loss.item(), 'cls_loss': cls_loss.item(), 'reg_loss': reg_loss.item() }
if __name__ == "__main__": model = MHLTDriverEmotionModel( video_feature_dim=512, audio_feature_dim=256, hidden_dim=256, num_heads=8 ) batch_size = 4 seq_len = 30 video_feat = torch.randn(batch_size, seq_len, 512) audio_feat = torch.randn(batch_size, seq_len, 256) cls_pred, int_pred = model(video_feat, audio_feat) print("=== MHLT模型测试 ===") print(f"输入: 视频{video_feat.shape} + 音频{audio_feat.shape}") print(f"分类输出: {cls_pred.shape} (3类: 正/中/负)") print(f"强度输出: {int_pred.shape} (范围: [-1, +1])") print(f"强度值: {int_pred.detach().numpy().flatten()}") trainer = MHLTTrainer(model) labels = torch.tensor([2, 1, 0, 2]) intensities = torch.tensor([-0.7, 0.0, 0.3, -0.8]) for epoch in range(5): result = trainer.train_step(video_feat, audio_feat, labels, intensities) print(f"Epoch {epoch+1}: loss={result['total_loss']:.4f} " f"(cls={result['cls_loss']:.4f}, reg={result['reg_loss']:.4f})") print("\n=== 情绪强度解读 ===") print("-0.8: 极端负面 → 可能危险驾驶(愤怒/恐慌)") print("-0.2: 轻度负面 → 可能更谨慎驾驶") print(" 0.0: 中性 → 正常驾驶状态") print("+0.3: 轻度正面 → 放松但可能分心") print("+0.8: 极端正面 → 兴奋可能导致冒险")
|