1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
| import torch import torch.nn as nn from typing import Dict, List
class SensAIEncoder(nn.Module): """ SensAI多模态编码器基类 设计原则: 1. 每个模态有独立编码器 2. 编码器输出标准化表示(统一维度) 3. 支持时序建模(LSTM/Transformer) 4. 缺失模态时仍可工作(降级运行) """ def __init__(self, input_dim: int, hidden_dim: int = 256, num_layers: int = 2, dropout: float = 0.1): super().__init__() self.hidden_dim = hidden_dim self.lstm = nn.LSTM( input_size=input_dim, hidden_size=hidden_dim, num_layers=num_layers, batch_first=True, dropout=dropout ) self.attention = nn.MultiheadAttention( embed_dim=hidden_dim, num_heads=4, batch_first=True ) self.reliability_head = nn.Sequential( nn.Linear(hidden_dim, hidden_dim // 2), nn.GELU(), nn.Linear(hidden_dim // 2, 1), nn.Sigmoid() ) def forward(self, x: torch.Tensor, mask: torch.Tensor = None) -> Dict[str, torch.Tensor]: """ Args: x: (B, T, input_dim) 模态输入序列 mask: (B, T) 有效帧掩码 Returns: encoded: (B, hidden_dim) 编码特征 reliability: (B, 1) 模态可靠性 """ lstm_out, _ = self.lstm(x) attn_out, _ = self.attention(lstm_out, lstm_out, lstm_out, key_padding_mask=mask) encoded = attn_out.mean(dim=1) reliability = self.reliability_head(encoded) return { 'encoded': encoded, 'reliability': reliability, 'temporal_features': lstm_out }
class VisualEncoder(SensAIEncoder): """视觉编码器:视线+姿态+活动""" def __init__(self, hidden_dim: int = 256): super().__init__(input_dim=128, hidden_dim=hidden_dim) self.gaze_proj = nn.Linear(4, 32) self.pose_proj = nn.Linear(34, 64) self.activity_proj = nn.Linear(10, 32)
class AcousticEncoder(SensAIEncoder): """声学编码器:场景分析+语音情感""" def __init__(self, hidden_dim: int = 256): super().__init__(input_dim=64, hidden_dim=hidden_dim) self.audio_proj = nn.Linear(64, 64)
class PhysiologicalEncoder(SensAIEncoder): """生理编码器:HRV+呼吸频率""" def __init__(self, hidden_dim: int = 256): super().__init__(input_dim=8, hidden_dim=hidden_dim) self.rppg_proj = nn.Linear(8, 32)
class VehicleEncoder(SensAIEncoder): """车辆数据编码器:方向盘+踏板+加速度""" def __init__(self, hidden_dim: int = 256): super().__init__(input_dim=12, hidden_dim=hidden_dim) self.can_proj = nn.Linear(12, 32)
class SensAIFusionModel(nn.Module): """ SensAI多模态融合模型 将4个模态编码器的输出融合为认知状态数字孪生 架构: 1. 4个独立编码器 2. 可靠性加权融合 3. 认知状态解码 4. 上下文敏感警告触发 """ def __init__(self, hidden_dim: int = 256, num_distraction_levels: int = 4): super().__init__() self.visual_encoder = VisualEncoder(hidden_dim) self.acoustic_encoder = AcousticEncoder(hidden_dim) self.physio_encoder = PhysiologicalEncoder(hidden_dim) self.vehicle_encoder = VehicleEncoder(hidden_dim) self.fusion = nn.Sequential( nn.Linear(hidden_dim * 4, hidden_dim * 2), nn.GELU(), nn.Dropout(0.2), nn.Linear(hidden_dim * 2, hidden_dim), nn.GELU(), nn.Dropout(0.2) ) self.state_decoder = nn.Sequential( nn.Linear(hidden_dim, hidden_dim), nn.GELU(), nn.Linear(hidden_dim, num_distraction_levels) ) self.factor_analyzer = nn.Linear(hidden_dim, 4) def forward(self, visual, acoustic, physio, vehicle, masks: Dict = None) -> Dict[str, torch.Tensor]: """ Args: visual: (B, T, 128) 视觉特征序列 acoustic: (B, T, 64) 声学特征序列 physio: (B, T, 8) 生理特征序列 vehicle: (B, T, 12) 车辆特征序列 masks: 各模态的有效帧掩码 Returns: distraction_logits: (B, 4) 分心等级 distraction_level: (B,) 预测等级 modality_weights: (B, 4) 各模态贡献度 """ vis_out = self.visual_encoder(visual, masks.get('visual') if masks else None) aco_out = self.acoustic_encoder(acoustic, masks.get('acoustic') if masks else None) phy_out = self.physio_encoder(physio, masks.get('physio') if masks else None) veh_out = self.vehicle_encoder(vehicle, masks.get('vehicle') if masks else None) vis_weighted = vis_out['encoded'] * vis_out['reliability'] aco_weighted = aco_out['encoded'] * aco_out['reliability'] phy_weighted = phy_out['encoded'] * phy_out['reliability'] veh_weighted = veh_out['encoded'] * veh_out['reliability'] concat = torch.cat([ vis_weighted, aco_weighted, phy_weighted, veh_weighted ], dim=-1) fused = self.fusion(concat) logits = self.state_decoder(fused) level = logits.argmax(dim=-1) weights = self.factor_analyzer(fused) weights = torch.softmax(weights, dim=-1) return { 'distraction_logits': logits, 'distraction_level': level, 'modality_weights': weights, 'reliabilities': { 'visual': vis_out['reliability'], 'acoustic': aco_out['reliability'], 'physio': phy_out['reliability'], 'vehicle': veh_out['reliability'] } }
if __name__ == "__main__": print("=" * 60) print("SensAI 多模态认知分心检测 - 模拟测试") print("=" * 60) model = SensAIFusionModel() total_params = sum(p.numel() for p in model.parameters()) print(f"\n模型参数量: {total_params:,}") B, T = 2, 30 visual = torch.randn(B, T, 128) acoustic = torch.randn(B, T, 64) physio = torch.randn(B, T, 8) vehicle = torch.randn(B, T, 12) with torch.no_grad(): output = model(visual, acoustic, physio, vehicle) print(f"\n输入:") print(f" 视觉: {visual.shape}") print(f" 声学: {acoustic.shape}") print(f" 生理: {physio.shape}") print(f" 车辆: {vehicle.shape}") print(f"\n输出:") print(f" 分心等级logits: {output['distraction_logits'].shape}") print(f" 预测等级: {output['distraction_level']}") print(f" 模态权重: {output['modality_weights']}") print(f"\n{'='*60}") print("模态降级测试(生理信号缺失)") print(f"{'='*60}") physio_mask = torch.ones(B, T) physio_mask[:, :] = 0 with torch.no_grad(): output_degraded = model(visual, acoustic, physio, vehicle, masks={'physio': physio_mask.bool()}) print(f" 视觉可靠性: {output_degraded['reliabilities']['visual'].mean():.3f}") print(f" 声学可靠性: {output_degraded['reliabilities']['acoustic'].mean():.3f}") print(f" 生理可靠性: {output_degraded['reliabilities']['physio'].mean():.3f}") print(f" 车辆可靠性: {output_degraded['reliabilities']['vehicle'].mean():.3f}") print(f"\n 降级时模态权重变化:") print(f" 正常: {output['modality_weights'][0]}") print(f" 降级: {output_degraded['modality_weights'][0]}")
|