1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
| import torch import torch.nn as nn
class InCarEmoFusionModel(nn.Module): """ InCarEmo 多模态融合模型 融合4个模态:面部+语音+生理+驾驶行为 """ def __init__(self, n_emotions=7): super().__init__() self.face_encoder = nn.Sequential( nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.ReLU(), nn.AdaptiveAvgPool2d((4, 4)), ) self.face_lstm = nn.LSTM(128*16, 256, batch_first=True) self.voice_encoder = nn.Sequential( nn.Conv1d(80, 128, 3, stride=2, padding=1), nn.ReLU(), nn.Conv1d(128, 256, 3, stride=2, padding=1), nn.ReLU(), nn.AdaptiveAvgPool1d(1), ) self.phys_encoder = nn.Sequential( nn.Linear(200, 128), nn.ReLU(), nn.Linear(128, 64), nn.ReLU(), ) self.drive_encoder = nn.Sequential( nn.Linear(120, 64), nn.ReLU(), nn.Linear(64, 32), nn.ReLU(), ) self.fusion = nn.Sequential( nn.Linear(256 + 256 + 64 + 32, 256), nn.ReLU(), nn.Dropout(0.3), nn.Linear(256, n_emotions) ) self.va_head = nn.Linear(256, 2) def forward(self, face, voice, phys, driving): b, t, c, h, w = face.shape face_feat = self.face_encoder(face.view(b*t, c, h, w)) face_feat = face_feat.view(b, t, -1) _, (hn, _) = self.face_lstm(face_feat) face_out = hn[-1] voice_out = self.voice_encoder(voice).squeeze(-1) phys_out = self.phys_encoder(phys) drive_out = self.drive_encoder(driving) combined = torch.cat([face_out, voice_out, phys_out, drive_out], dim=1) emotions = self.fusion(combined) va = self.va_head(combined) return { 'emotions': emotions, 'valence_arousal': va, }
if __name__ == "__main__": model = InCarEmoFusionModel() face = torch.randn(2, 30, 3, 64, 64) voice = torch.randn(2, 80, 160) phys = torch.randn(2, 200) driving = torch.randn(2, 120) output = model(face, voice, phys, driving) print(f"情绪分类: {output['emotions'].shape}") print(f"效价/唤醒度: {output['valence_arousal'].shape}") print(f"参数总数: {sum(p.numel() for p in model.parameters()):,}")
|