1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137
| import torch import torch.nn as nn import torch.nn.functional as F
class CameraRadarFusion(nn.Module): """ Murata+SmartEye风格的摄像头-雷达融合模型 核心思路: 1. 摄像头提供视觉上下文(姿态、行为、表情) 2. 雷达提供深度和穿透检测(存在、距离、微动) 3. 融合层学习两个模态的互补关系 支持任务:DMS, OMS, CPD, OOP, 生命体征 """ def __init__(self, n_tasks=5, d_model=256): super().__init__() self.camera_encoder = CameraEncoder(d_model=d_model) self.radar_encoder = RadarEncoder(d_model=d_model) self.cross_attention = nn.MultiheadAttention( d_model, num_heads=8, batch_first=True ) self.task_heads = nn.ModuleDict({ 'dms_gaze': nn.Linear(d_model, 4), 'dms_fatigue': nn.Linear(d_model, 3), 'dms_distraction': nn.Linear(d_model, 3), 'oms_occupant': nn.Linear(d_model, 4), 'oms_posture': nn.Linear(d_model, 8), 'cpd_presence': nn.Linear(d_model, 2), 'cpd_breathing': nn.Linear(d_model, 1), 'oop_detection': nn.Linear(d_model, 5), 'airbag_deploy': nn.Linear(d_model, 3), }) def forward(self, camera_img, radar_rd): """ Args: camera_img: (B, 3, H, W) 座舱图像 radar_rd: (B, 1, R, D) range-Doppler图 Returns: dict of task outputs """ cam_feat = self.camera_encoder(camera_img) rad_feat = self.radar_encoder(radar_rd) cam_stack = cam_feat.unsqueeze(1) rad_stack = rad_feat.unsqueeze(1) cam_enhanced, _ = self.cross_attention(cam_stack, rad_stack, rad_stack) cam_enhanced = cam_enhanced.squeeze(1) fused = cam_feat + rad_feat + cam_enhanced outputs = {} for task_name, head in self.task_heads.items(): outputs[task_name] = head(fused) return outputs
class CameraEncoder(nn.Module): """摄像头编码器(MobileNet风格轻量化)""" def __init__(self, d_model=256): super().__init__() self.features = nn.Sequential( nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.BatchNorm2d(32), nn.ReLU6(), nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.BatchNorm2d(64), nn.ReLU6(), nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.BatchNorm2d(128), nn.ReLU6(), nn.Conv2d(128, d_model, 3, stride=2, padding=1), nn.BatchNorm2d(d_model), nn.ReLU6(), nn.AdaptiveAvgPool2d(1), ) self.proj = nn.Linear(d_model, d_model) def forward(self, x): return self.proj(self.features(x).flatten(1))
class RadarEncoder(nn.Module): """60GHz雷达 range-Doppler 编码器""" def __init__(self, d_model=256): super().__init__() self.features = nn.Sequential( nn.Conv2d(1, 32, 3, padding=1), nn.BatchNorm2d(32), nn.ReLU(), nn.MaxPool2d(2), nn.Conv2d(32, 64, 3, padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.MaxPool2d(2), nn.Conv2d(64, 128, 3, padding=1), nn.BatchNorm2d(128), nn.ReLU(), nn.MaxPool2d(2), nn.Conv2d(128, d_model, 3, padding=1), nn.BatchNorm2d(d_model), nn.ReLU(), nn.AdaptiveAvgPool2d(1), ) self.proj = nn.Linear(d_model, d_model) def forward(self, x): return self.proj(self.features(x).flatten(1))
if __name__ == "__main__": model = CameraRadarFusion(n_tasks=5, d_model=128) B = 2 camera = torch.randn(B, 3, 224, 224) radar = torch.randn(B, 1, 64, 64) outputs = model(camera, radar) print("=== 摄像头-雷达融合多任务输出 ===") for task, out in outputs.items(): print(f" {task}: {out.shape}") total_params = sum(p.numel() for p in model.parameters()) print(f"\n参数量: {total_params/1e6:.2f}M") print(f"适合边缘部署: {'✅' if total_params < 50e6 else '⚠️ 需量化'}")
|