1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
| import torch import torch.nn as nn from typing import Dict
class ModalityEncoder(nn.Module): """单模态编码器基类""" def __init__(self, input_dim: int, hidden_dim: int = 128): super().__init__() self.encoder = nn.Sequential( nn.Linear(input_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU(), nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU() ) def forward(self, x: torch.Tensor) -> torch.Tensor: return self.encoder(x)
class VisualEncoder(ModalityEncoder): """ 视觉模态编码器 输入特征向量: - gaze_xy: (2,) 视线方向 - gaze_confidence: (1,) 置信度 - head_pose: (6,) 头部姿态 (pitch, yaw, roll + 角速度) - body_activity: (4,) 身体活动量 (4个身体部位) - interaction_score: (3,) 物体交互评分 (手机/中控/方向盘) 总输入维度: 16 """ def __init__(self, hidden_dim: int = 128): super().__init__(input_dim=16, hidden_dim=hidden_dim) self.temporal = nn.LSTM( input_size=hidden_dim, hidden_size=hidden_dim, num_layers=2, batch_first=True, bidirectional=False ) def forward(self, x: torch.Tensor) -> torch.Tensor: """ Args: x: (B, T, 16) 时序视觉特征 Returns: encoded: (B, hidden_dim) 模态级嵌入 """ B, T, _ = x.shape feat = self.encoder(x.reshape(B * T, -1)) feat = feat.reshape(B, T, -1) _, (hn, _) = self.temporal(feat) return hn[-1]
class AudioEncoder(ModalityEncoder): """ 音频模态编码器 输入特征向量: - scene_embedding: (8,) 声学场景嵌入 - speech_features: (6,) 语音韵律 (语速/停顿/音高/能量) - breathing_rate: (2,) 呼吸频率和方差 - attention_score: (1,) 注意力声学评分 总输入维度: 17 """ def __init__(self, hidden_dim: int = 128): super().__init__(input_dim=17, hidden_dim=hidden_dim) self.temporal = nn.LSTM( input_size=hidden_dim, hidden_size=hidden_dim, num_layers=1, batch_first=True ) def forward(self, x: torch.Tensor) -> torch.Tensor: B, T, _ = x.shape feat = self.encoder(x.reshape(B * T, -1)).reshape(B, T, -1) _, (hn, _) = self.temporal(feat) return hn[-1]
class VitalSignEncoder(ModalityEncoder): """ 生命体征模态编码器 输入特征向量: - heart_rate: (2,) 心率和变化率 - hrv_features: (5,) HRV时域频域特征 - eda_features: (3,) 皮肤电活动特征 - respiration: (2,) 呼吸率和方差 总输入维度: 12 """ def __init__(self, hidden_dim: int = 128): super().__init__(input_dim=12, hidden_dim=hidden_dim) self.temporal = nn.LSTM( input_size=hidden_dim, hidden_size=hidden_dim, num_layers=1, batch_first=True ) def forward(self, x: torch.Tensor) -> torch.Tensor: B, T, _ = x.shape feat = self.encoder(x.reshape(B * T, -1)).reshape(B, T, -1) _, (hn, _) = self.temporal(feat) return hn[-1]
class VehicleDataEncoder(ModalityEncoder): """ 车辆数据模态编码器 输入特征向量: - steering_micro: (3,) 方向盘微操作 (角度/速度/加速度) - pedal_pattern: (2,) 踏板操作平稳度 - lane_keeping: (2,) 车道偏移和方差 - headway: (2,) 前车距离和变化率 总输入维度: 9 """ def __init__(self, hidden_dim: int = 128): super().__init__(input_dim=9, hidden_dim=hidden_dim) self.temporal = nn.LSTM( input_size=hidden_dim, hidden_size=hidden_dim, num_layers=1, batch_first=True ) def forward(self, x: torch.Tensor) -> torch.Tensor: B, T, _ = x.shape feat = self.encoder(x.reshape(B * T, -1)).reshape(B, T, -1) _, (hn, _) = self.temporal(feat) return hn[-1]
class SensAIFusionModel(nn.Module): """ SensAI 多模态融合模型 架构: 1. 四个专用编码器提取各模态嵌入 2. Cross-Attention 融合层实现跨模态交互 3. 认知状态解码器输出认知分心评估 输出: - cognitive_state: (B, 3) [正常, 轻度分心, 严重分心] 概率 - contributing_factors: (B, 4) 各模态贡献度 """ def __init__(self, hidden_dim: int = 128, num_heads: int = 4): super().__init__() self.visual_enc = VisualEncoder(hidden_dim) self.audio_enc = AudioEncoder(hidden_dim) self.vital_enc = VitalSignEncoder(hidden_dim) self.vehicle_enc = VehicleDataEncoder(hidden_dim) self.cross_attn = nn.MultiheadAttention( embed_dim=hidden_dim, num_heads=num_heads, batch_first=True ) self.modality_weights = nn.Sequential( nn.Linear(hidden_dim * 4, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, 4), nn.Softmax(dim=-1) ) self.state_decoder = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.LayerNorm(hidden_dim), nn.GELU(), nn.Linear(hidden_dim, 3), nn.Softmax(dim=-1) ) self.factor_explainer = nn.Sequential( nn.Linear(hidden_dim * 4, hidden_dim), nn.ReLU(), nn.Linear(hidden_dim, 4), nn.Sigmoid() ) def forward(self, visual: torch.Tensor, audio: torch.Tensor, vital: torch.Tensor, vehicle: torch.Tensor) -> dict: """ Args: visual: (B, T, 16) audio: (B, T, 17) vital: (B, T, 12) vehicle: (B, T, 9) Returns: dict with 'cognitive_state', 'contributing_factors' """ v_emb = self.visual_enc(visual) a_emb = self.audio_enc(audio) s_emb = self.vital_enc(vital) c_emb = self.vehicle_enc(vehicle) stacked = torch.stack([v_emb, a_emb, s_emb, c_emb], dim=1) fused, _ = self.cross_attn(stacked, stacked, stacked) fused_repr = fused.mean(dim=1) concat_emb = torch.cat([v_emb, a_emb, s_emb, c_emb], dim=-1) weights = self.modality_weights(concat_emb) weighted_emb = (stacked * weights.unsqueeze(-1)).sum(dim=1) final_repr = fused_repr + weighted_emb cognitive_state = self.state_decoder(final_repr) contributing_factors = self.factor_explainer(concat_emb) return { 'cognitive_state': cognitive_state, 'contributing_factors': contributing_factors, 'modality_weights': weights }
if __name__ == "__main__": model = SensAIFusionModel(hidden_dim=64, num_heads=4) B, T = 2, 30 visual = torch.randn(B, T, 16) audio = torch.randn(B, T, 17) vital = torch.randn(B, T, 12) vehicle = torch.randn(B, T, 9) output = model(visual, audio, vital, vehicle) print("=== SensAI 融合模型输出 ===") print(f"认知状态: {output['cognitive_state'].shape}") print(f" 正常概率: {output['cognitive_state'][0, 0]:.3f}") print(f" 轻度分心: {output['cognitive_state'][0, 1]:.3f}") print(f" 严重分心: {output['cognitive_state'][0, 2]:.3f}") print(f"\n模态权重: {output['modality_weights'][0].tolist()}") print(f" 视觉: {output['modality_weights'][0, 0]:.3f}") print(f" 音频: {output['modality_weights'][0, 1]:.3f}") print(f" 生命体征: {output['modality_weights'][0, 2]:.3f}") print(f" 车辆: {output['modality_weights'][0, 3]:.3f}") total_params = sum(p.numel() for p in model.parameters()) print(f"\n模型参数量: {total_params:,}") print(f"模型大小: {total_params * 4 / 1024 / 1024:.1f} MB (FP32)")
|