1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
| class GNNEmotionRecognizer(nn.Module):
"""
GNN 多模态情感识别器
=====================
论文 Section 4: Model Architecture
架构:
1. 模态编码器 (各模态独立)
2. 图构建 (多模态图)
3. GNN 层 (消息传递 + 聚合)
4. 图级池化
5. 情感分类头
Args:
eeg_dim: EEG 输入维度
face_dim: 面部输入维度
phys_dim: 生理信号输入维度
hidden_dim: GNN 隐藏维度
num_emotions: 情感类别数
num_gnn_layers: GNN 层数
heads: 注意力头数
"""
def __init__(
self,
eeg_dim: int = 128,
face_dim: int = 256,
phys_dim: int = 32,
hidden_dim: int = 128,
num_emotions: int = 7,
num_gnn_layers: int = 3,
heads: int = 4,
dropout: float = 0.1
):
super().__init__()
self.eeg_encoder = nn.Sequential(
nn.Linear(eeg_dim, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.GELU(),
nn.Dropout(dropout)
)
self.face_encoder = nn.Sequential(
nn.Linear(face_dim, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.GELU(),
nn.Dropout(dropout)
)
self.phys_encoder = nn.Sequential(
nn.Linear(phys_dim, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.GELU(),
nn.Dropout(dropout)
)
self.gnn_layers = nn.ModuleList([
GATConv(hidden_dim, hidden_dim // heads, heads=heads, dropout=dropout)
for _ in range(num_gnn_layers)
])
self.gnn_norms = nn.ModuleList([
nn.LayerNorm(hidden_dim) for _ in range(num_gnn_layers)
])
self.modality_embed = nn.Embedding(3, hidden_dim)
self.classifier = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim),
nn.LayerNorm(hidden_dim),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(hidden_dim, num_emotions)
)
self.regressor = nn.Sequential(
nn.Linear(hidden_dim, hidden_dim // 2),
nn.GELU(),
nn.Linear(hidden_dim // 2, 2)
)
def forward(
self,
eeg_feat: torch.Tensor,
face_feat: torch.Tensor,
phys_feat: torch.Tensor,
edge_index: torch.Tensor,
node_types: torch.Tensor,
batch: torch.Tensor
) -> Dict[str, torch.Tensor]:
"""
前向传播
Args:
eeg_feat: [N_eeg, eeg_dim] EEG节点特征
face_feat: [N_face, face_dim] 面部节点特征
phys_feat: [N_phys, phys_dim] 生理节点特征
edge_index: [2, E] 边索引
node_types: [N] 节点类型 (0=EEG, 1=Face, 2=Phys)
batch: [N] batch索引
Returns:
emotions: [B, num_emotions] 情感分类
va: [B, 2] 效价-唤醒度
"""
eeg_h = self.eeg_encoder(eeg_feat)
face_h = self.face_encoder(face_feat)
phys_h = self.phys_encoder(phys_feat)
x = torch.cat([eeg_h, face_h, phys_h], dim=0)
x = x + self.modality_embed(node_types)
for gnn, norm in zip(self.gnn_layers, self.gnn_norms):
residual = x
x = gnn(x, edge_index)
x = norm(x + residual)
graph_emb = global_mean_pool(x, batch)
emotions = self.classifier(graph_emb)
va = self.regressor(graph_emb)
return {
"emotions": emotions,
"valence_arousal": va,
"graph_embedding": graph_emb
}
if __name__ == "__main__":
model = GNNEmotionRecognizer(
eeg_dim=128,
face_dim=256,
phys_dim=32,
hidden_dim=128,
num_emotions=7,
num_gnn_layers=3,
heads=4
)
total_params = sum(p.numel() for p in model.parameters())
print(f"GNN 情感识别器")
print(f"总参数: {total_params:,} ({total_params/1e6:.2f}M)")
print(f"GNN 层数: 3")
print(f"注意力头: 4")
print(f"情感类别: 7 (高兴/悲伤/愤怒/恐惧/惊讶/厌恶/中性)")
|
