1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
| """ HelloWorld 驾驶世界模型核心架构复现
关键组件: 1. 2B参数世界基础模型 2. Block-Causal 生成接口 3. 7摄像头同步生成 4. 条件LiDAR合成 5. 少步推理蒸馏
论文亮点:从异构视频数据学习广义先验, 然后逐步专业化到可控驾驶场景生成 """
import torch import torch.nn as nn from typing import List, Optional, Tuple
class DrivingWorldModel(nn.Module): """ 2B参数驾驶世界模型 输入条件: - ego_pose: 自车位姿 (x, y, theta) - hd_map: HD地图栅格化表示 - 3d_boxes: 3D目标边界框 输出: - 7摄像头RGB图像 - LiDAR点云 """ def __init__(self, latent_dim=256, num_cameras=7, map_channels=3, map_size=256, hidden_dim=1024, num_layers=24): super().__init__() self.latent_dim = latent_dim self.num_cameras = num_cameras self.pose_encoder = nn.Sequential( nn.Linear(3, 64), nn.ReLU(), nn.Linear(64, latent_dim) ) self.map_encoder = nn.Conv2d( map_channels, latent_dim // 4, kernel_size=8, stride=4 ) self.box_encoder = nn.Sequential( nn.Linear(7, 64), nn.ReLU(), nn.Linear(64, latent_dim) ) self.backbone = nn.TransformerEncoder( nn.TransformerEncoderLayer( d_model=hidden_dim, nhead=16, dim_feedforward=hidden_dim * 4, dropout=0.1, batch_first=True ), num_layers=num_layers ) self.camera_decoders = nn.ModuleList([ CameraDecoder(hidden_dim, out_channels=3) for _ in range(num_cameras) ]) self.lidar_decoder = LiDARDecoder(hidden_dim) def forward(self, ego_pose, hd_map, boxes_3d, context_frames=None): """ 前向传播 Args: ego_pose: (B, 3) 自车位姿 hd_map: (B, C, H, W) HD地图 boxes_3d: (B, N, 7) 3D边界框 context_frames: (B, T, ...) 历史上下文 Returns: cameras: (B, 7, 3, H, W) 7摄像头RGB lidar: (B, P, 4) LiDAR点云 """ pose_feat = self.pose_encoder(ego_pose) map_feat = self.map_encoder(hd_map) map_feat = map_feat.flatten(2).mean(dim=2) B, N, _ = boxes_3d.shape box_feat = self.box_encoder(boxes_3d) box_feat = box_feat.mean(dim=1) cond = pose_feat + map_feat + box_feat if context_frames is not None: T = context_frames.shape[1] context_emb = self._encode_context(context_frames) seq = torch.cat([context_emb, cond.unsqueeze(1)], dim=1) else: seq = cond.unsqueeze(1) out = self.backbone(seq) world_state = out[:, -1, :] cameras = [] for decoder in self.camera_decoders: cam_img = decoder(world_state) cameras.append(cam_img) cameras = torch.stack(cameras, dim=1) lidar = self.lidar_decoder(world_state) return cameras, lidar def _encode_context(self, frames): """编码历史上下文帧""" B, T = frames.shape[0], frames.shape[1] flat = frames.reshape(B, T, -1) return nn.Linear(flat.shape[-1], 1024).to(flat.device)(flat)
class CameraDecoder(nn.Module): """单摄像头解码器""" def __init__(self, in_dim, out_channels=3): super().__init__() self.net = nn.Sequential( nn.Linear(in_dim, 512), nn.ReLU(), nn.Linear(512, 256 * 4 * 4), nn.ReLU(), nn.Unflatten(1, (256, 4, 4)), nn.ConvTranspose2d(256, 128, 4, 2, 1), nn.ReLU(), nn.ConvTranspose2d(128, 64, 4, 2, 1), nn.ReLU(), nn.ConvTranspose2d(64, 32, 4, 2, 1), nn.ReLU(), nn.ConvTranspose2d(32, out_channels, 4, 2, 1), ) def forward(self, x): return self.net(x)
class LiDARDecoder(nn.Module): """LiDAR点云解码器""" def __init__(self, in_dim, num_points=16384): super().__init__() self.num_points = num_points self.net = nn.Sequential( nn.Linear(in_dim, 512), nn.ReLU(), nn.Linear(512, num_points * 4), ) def forward(self, x): B = x.shape[0] out = self.net(x) return out.reshape(B, self.num_points, 4)
if __name__ == "__main__": model = DrivingWorldModel( latent_dim=256, num_cameras=7, hidden_dim=1024, num_layers=24 ) total_params = sum(p.numel() for p in model.parameters()) print(f"=== HelloWorld 模型 ===") print(f" 总参数量: {total_params / 1e9:.2f}B") print(f" 摄像头数: 7") print(f" LiDAR: 条件合成") B = 2 ego_pose = torch.randn(B, 3) hd_map = torch.randn(B, 3, 256, 256) boxes_3d = torch.randn(B, 20, 7) cameras, lidar = model(ego_pose, hd_map, boxes_3d) print(f"\n=== 推理结果 ===") print(f" 摄像头输出: {cameras.shape}") print(f" LiDAR输出: {lidar.shape}") print(f" 摄像头数: {cameras.shape[1]}")
|