Dapalm | IMS技术前沿

三模态情感识别：CNN+BiLSTM+ECAPA-TDNN融合框架——疲劳/压力/分心检测的新架构

论文信息

项	详情
标题	Robust Multimodal Sentiment Recognition Using Facial Expressions, Heart Rate Variability and Speech Signals
来源	Research Square (2026)
核心	面部表情CNN + HRV BiLSTM + 语音ECAPA-TDNN 三模态融合

核心创新

传统单模态情感识别在复杂环境下鲁棒性不足。本文提出三模态互补融合框架：

# 三模态情感识别融合架构
import torch
import torch.nn as nn

class MultimodalSentimentRecognition(nn.Module):
    """
    三模态情感识别
    
    模态1: 面部表情 → CNN (ResNet50-based)
    模态2: HRV信号 → BiLSTM
    模态3: 语音情感 → ECAPA-TDNN
    
    融合: Late Fusion (加权平均 + 注意力)
    """
    
    def __init__(self, num_classes=7, hidden_dim=256):
        super().__init__()
        
        # 模态1: 面部表情 CNN
        self.face_cnn = nn.Sequential(
            nn.Conv2d(3, 64, 7, stride=2, padding=3),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(3, stride=2, padding=1),
            self._make_res_block(64, 128, stride=2),
            self._make_res_block(128, 256, stride=2),
            self._make_res_block(256, 512, stride=2),
            nn.AdaptiveAvgPool2d((1, 1)),
            nn.Flatten(),
            nn.Linear(512, hidden_dim),
        )
        
        # 模态2: HRV BiLSTM
        self.hrv_bilstm = nn.LSTM(
            input_size=5,  # RR间期/SDNN/RMSSD/pNN50/LF_HF
            hidden_size=hidden_dim,
            num_layers=2,
            batch_first=True,
            bidirectional=True,
        )
        self.hrv_fc = nn.Linear(hidden_dim * 2, hidden_dim)
        
        # 模态3: 语音 ECAPA-TDNN
        self.speech_tdnn = ECAPA_TDNN(
            input_dim=80,  # MFCC特征
            hidden_dim=hidden_dim,
            num_blocks=3,
        )
        
        # 融合层
        self.attention = nn.MultiheadAttention(
            embed_dim=hidden_dim, num_heads=8
        )
        self.classifier = nn.Sequential(
            nn.Linear(hidden_dim * 3, hidden_dim),
            nn.ReLU(),
            nn.Dropout(0.3),
            nn.Linear(hidden_dim, num_classes),
        )
    
    def _make_res_block(self, in_ch, out_ch, stride=1):
        return nn.Sequential(
            nn.Conv2d(in_ch, out_ch, 3, stride=stride, padding=1),
            nn.BatchNorm2d(out_ch),
            nn.ReLU(),
            nn.Conv2d(out_ch, out_ch, 3, padding=1),
            nn.BatchNorm2d(out_ch),
        )
    
    def forward(self, face, hrv, speech):
        # 模态1: 面部
        face_feat = self.face_cnn(face)  # (B, hidden_dim)
        
        # 模态2: HRV
        hrv_out, _ = self.hrv_bilstm(hrv)  # (B, T, 2*hidden)
        hrv_feat = self.hrv_fc(hrv_out[:, -1, :])  # (B, hidden_dim)
        
        # 模态3: 语音
        speech_feat = self.speech_tdnn(speech)  # (B, hidden_dim)
        
        # 融合
        combined = torch.cat([face_feat, hrv_feat, speech_feat], dim=1)
        output = self.classifier(combined)
        return output


class ECAPA_TDNN(nn.Module):
    """ECAPA-TDNN 语音特征提取器"""
    
    def __init__(self, input_dim=80, hidden_dim=256, num_blocks=3):
        super().__init__()
        self.blocks = nn.ModuleList([
            nn.Sequential(
                nn.Conv1d(input_dim if i == 0 else hidden_dim, hidden_dim, 5, padding=2),
                nn.BatchNorm1d(hidden_dim),
                nn.ReLU(),
                nn.SEBlock(hidden_dim),
            ) for i in range(num_blocks)
        ])
        self.att_pool = nn.MultiheadAttention(hidden_dim, 8)
    
    def forward(self, x):
        for block in self.blocks:
            x = block(x)
        return x.mean(dim=2)  # (B, hidden_dim)


class SEBlock(nn.Module):
    """Squeeze-and-Excitation Block"""
    def __init__(self, channels, reduction=16):
        super().__init__()
        self.fc = nn.Sequential(
            nn.Linear(channels, channels // reduction),
            nn.ReLU(),
            nn.Linear(channels // reduction, channels),
            nn.Sigmoid(),
        )
    
    def forward(self, x):
        b, c, _ = x.shape
        w = x.mean(dim=2)
        w = self.fc(w).unsqueeze(2)
        return x * w

模态对比

模态	传感器	特征	优势	局限
面部表情	RGB/NIR摄像头	表情单元+微表情	直观/低成本	光照敏感/遮挡
HRV	方向盘ECG/rPPG	RR间期/SDNN/RMSSD	生理客观	噪声/个体差异
语音	麦克风	MFCC/韵律/音色	非接触/低成本	环境噪声/语言依赖

IMS 应用映射

疲劳/压力/分心检测映射

# 情感→IMS状态映射
emotion_to_ims = {
    "happy": {"fatigue": 0.1, "stress": 0.1, "distraction": 0.1},
    "sad": {"fatigue": 0.3, "stress": 0.3, "distraction": 0.2},
    "angry": {"fatigue": 0.1, "stress": 0.8, "distraction": 0.5},
    "fear": {"fatigue": 0.2, "stress": 0.9, "distraction": 0.6},
    "surprise": {"fatigue": 0.1, "stress": 0.3, "distraction": 0.7},
    "disgust": {"fatigue": 0.2, "stress": 0.5, "distraction": 0.3},
    "neutral": {"fatigue": 0.2, "stress": 0.2, "distraction": 0.2},
}

# IMS 决策策略
def ims_decision(emotion, dms_state):
    stress = emotion_to_ims[emotion]["stress"]
    fatigue = dms_state["fatigue"]
    
    if stress > 0.7 and fatigue > 0.5:
        return "high_risk_warning"  # 高压力+高疲劳
    elif stress > 0.5:
        return "stress_management"  # 压力管理提示
    elif fatigue > 0.6:
        return "fatigue_warning"    # 疲劳警告

实验

# 复现评估
if __name__ == "__main__":
    model = MultimodalSentimentRecognition(num_classes=7)
    
    # 模拟输入
    face = torch.randn(4, 3, 224, 224)    # 4张224x224面部图
    hrv = torch.randn(4, 60, 5)           # 4个60秒HRV序列
    speech = torch.randn(4, 80, 200)     # 4段200帧语音
    
    output = model(face, hrv, speech)
    print(f"Output shape: {output.shape}")  # (4, 7)
    print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")

参考

• 论文: https://www.researchsquare.com/article/rs-10393694/v1