NVIDIA Omniverse NuRec + Cosmos:从真实传感器数据到无限场景生成

NVIDIA robotaxi 技术栈深度解读 — 合成数据对座舱感知的启示

新闻信息

核心架构:三计算机系统

NVIDIA 将 robotaxi 开发定义为”三计算机系统”:

NuRec:神经场景重建

核心能力

NVIDIA Omniverse NuRec 模型可以从现有传感器数据重建真实驾驶场景:

“Omniverse NuRec models can reconstruct real-world driving scenes from sensor data”
— NVIDIA, 2026.09.25

对座舱数据合成的意义:

  • 不需要从零建模座舱场景
  • 从真实 DMS/OMS 摄像头数据 → 重建座舱 3D 场景
  • 可以修改视角、光照、乘员位置等参数
  • 生成大量 corner case 训练数据
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
"""
NVIDIA NuRec 概念流程:从真实传感器数据到合成场景

虽然 NuRec 具体模型未开源,但可以复现其概念流程:
1. 多传感器数据采集
2. 神经辐射场(NeRF/3DGS)重建
3. 场景编辑(视角/光照/对象变换)
4. 新视角渲染
"""

import numpy as np
from dataclasses import dataclass, field
from typing import List, Optional
from enum import Enum


class SensorType(Enum):
RGB_CAMERA = "rgb_camera"
IR_CAMERA = "ir_camera"
RADAR = "radar"
LIDAR = "lidar"
UWB = "uwb"


@dataclass
class SensorFrame:
"""单帧传感器数据"""
sensor_type: SensorType
timestamp: float
data: np.ndarray # 图像/点云/雷达数据
extrinsics: np.ndarray # 4x4 外参矩阵
intrinsics: Optional[np.ndarray] = None # 3x3 内参矩阵


class NeuralReconstructionPipeline:
"""
NuRec 概念流程:神经场景重建管道

步骤:
1. 传感器数据预处理
2. 位姿估计(COLMAP/SfM)
3. 神经辐射场训练(3D Gaussian Splatting)
4. 场景编辑
5. 新视角渲染
"""

def __init__(self, scene_bounds: tuple = (0, 5, 0, 3, 0, 2.5)):
"""
Args:
scene_bounds: 座舱场景边界 (x_min, x_max, y_min, y_max, z_min, z_max)
典型座舱:5m x 3m x 2.5m
"""
self.scene_bounds = scene_bounds
self.frames: List[SensorFrame] = []
self.point_cloud = None
self.gaussian_model = None

def add_frame(self, frame: SensorFrame):
"""添加传感器帧"""
self.frames.append(frame)

def estimate_poses(self):
"""
多视角位姿估计

使用 COLMAP 风格的 SfM(Structure from Motion)
"""
num_frames = len(self.frames)
print(f" 帧数: {num_frames}")

# 模拟位姿估计
poses = []
for i, frame in enumerate(self.frames):
# 模拟相机位姿(围绕座舱轨道)
angle = 2 * np.pi * i / num_frames
t = np.array([
2.5 + 1.5 * np.cos(angle),
1.5 + 0.5 * np.sin(angle),
1.2
])

R = np.eye(4)
R[:3, :3] = np.array([
[np.cos(angle), 0, np.sin(angle)],
[0, 1, 0],
[-np.sin(angle), 0, np.cos(angle)]
])
R[:3, 3] = t

poses.append(R)

print(f" 估计位姿: {len(poses)} 个")
return poses

def train_gaussian_model(self, poses):
"""
3D Gaussian Splatting 训练

概念模型:
- 每个高斯球:位置 + 协方差 + 不透明度 + 球谐系数
- 渲染:alpha 混合
"""
# 模拟初始化 50000 个高斯球
num_gaussians = 50000

# 随机初始化在座舱空间内
positions = np.random.uniform(
low=[self.scene_bounds[0], self.scene_bounds[2], self.scene_bounds[4]],
high=[self.scene_bounds[1], self.scene_bounds[3], self.scene_bounds[5]],
size=(num_gaussians, 3)
)

# 协方差(各向同性初始化)
scales = np.random.uniform(0.001, 0.01, size=(num_gaussians, 3))
rotations = np.tile(np.array([1, 0, 0, 0]), (num_gaussians, 1))

# 不透明度
opacities = np.random.uniform(0.01, 0.5, size=num_gaussians)

# 球谐系数(模拟颜色)
sh_coeffs = np.random.randn(num_gaussians, 3, 3) # 3阶球谐

print(f" 高斯球数: {num_gaussians}")
print(f" 训练迭代: ~30000 steps")

return {
'positions': positions,
'scales': scales,
'rotations': rotations,
'opacities': opacities,
'sh_coeffs': sh_coeffs
}

def render_novel_view(self, model, camera_pose: np.ndarray,
resolution=(1920, 1080)):
"""
从新视角渲染场景

Args:
model: 训练好的高斯模型
camera_pose: 4x4 相机位姿
resolution: (width, height)

Returns:
rendered_image: shape=(H, W, 3)
"""
H, W = resolution[1], resolution[0]

# 模拟渲染(实际使用 CUDA 加速的光栅化)
image = np.zeros((H, W, 3), dtype=np.float32)

# 简化的 alpha 混合渲染
positions = model['positions']
opacities = model['opacities']
sh_coeffs = model['sh_coeffs']

# 投影到图像平面(简化版)
fx = fy = 1000 # 焦距
cx, cy = W/2, H/2

camera_center = camera_pose[:3, 3]
camera_R = camera_pose[:3, :3]

for i in range(0, len(positions), 100): # 采样渲染
world_pos = positions[i]
cam_pos = camera_R.T @ (world_pos - camera_center)

if cam_pos[2] <= 0:
continue

# 投影
u = int(fx * cam_pos[0] / cam_pos[2] + cx)
v = int(fy * cam_pos[1] / cam_pos[2] + cy)

if 0 <= u < W and 0 <= v < H:
color = np.clip(sh_coeffs[i, :, 0], 0, 1)
alpha = opacities[i]
image[v, u] = image[v, u] * (1 - alpha) + color * alpha

return image


# ========== 测试 ==========

if __name__ == "__main__":
pipeline = NeuralReconstructionPipeline()

# 模拟添加 20 帧 DMS 摄像头数据
for i in range(20):
frame = SensorFrame(
sensor_type=SensorType.RGB_CAMERA,
timestamp=i * 0.1,
data=np.random.randn(1080, 1920, 3),
extrinsics=np.eye(4)
)
pipeline.add_frame(frame)

print("=== NuRec 神经重建流程 ===")
print("Step 1: 位姿估计")
poses = pipeline.estimate_poses()

print("\nStep 2: 3D Gaussian Splatting 训练")
model = pipeline.train_gaussian_model(poses)

print("\nStep 3: 新视角渲染")
novel_pose = np.eye(4)
novel_pose[:3, 3] = np.array([2.0, 1.5, 1.3])
image = pipeline.render_novel_view(model, novel_pose)
print(f" 渲染图像: {image.shape}")
print(f" 非零像素: {np.count_nonzero(image.any(axis=2))}")

Cosmos 世界模型:场景变体生成

核心能力

“Cosmos world foundation models can generate variations in traffic, weather, lighting and sensor conditions”
— NVIDIA, 2026.09.25

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
"""
NVIDIA Cosmos 世界模型概念框架

Cosmos 是"世界基础模型"(World Foundation Model):
- 从真实数据学习世界动态规律
- 可以生成新的场景变体
- 支持条件生成(天气/光照/交通/传感器)

对座舱数据的启示:
- 从真实 DMS 数据生成新的疲劳/分心/酒驾场景
- 生成不同光照/遮挡/乘员组合
- 扩展训练数据集
"""

from dataclasses import dataclass
from typing import List, Optional
import numpy as np


@dataclass
class CabinSceneConfig:
"""座舱场景配置"""
# 乘员配置
driver_present: bool = True
driver_state: str = "normal" # normal/fatigue/distraction/alcohol
passenger_front: bool = False
passenger_rear_left: bool = False
passenger_rear_right: bool = False
child_seat: bool = False

# 环境条件
lighting: str = "day" # day/night/dusk/tunnel
temperature: float = 22.0 # 摄氏度

# 传感器配置
camera_position: str = "dashboard" # dashboard/overhead/A-pillar
camera_type: str = "rgb-ir" # rgb/rgb-ir/depth
radar_enabled: bool = True

# 场景动态
vehicle_speed: float = 60.0 # km/h
road_type: str = "highway" # highway/urban/rural


class CosmosCabinGenerator:
"""
Cosmos 座舱场景生成器概念

使用世界模型生成多样化的座舱场景
"""

def __init__(self):
self.base_scenes = []
self.generated_count = 0

def add_base_scene(self, config: CabinSceneConfig):
"""添加真实采集的基础场景"""
self.base_scenes.append(config)

def generate_variants(self, base: CabinSceneConfig,
num_variants: int = 10) -> List[CabinSceneConfig]:
"""
从基础场景生成变体

策略:
1. 改变环境条件(光照/温度)
2. 改变乘员组合
3. 改变驾驶员状态
4. 改变传感器配置
"""
variants = []

# 1. 光照变体
lightings = ["day", "night", "dusk", "tunnel", "dawn"]
for light in lightings:
if light != base.lighting:
v = CabinSceneConfig(**{**base.__dict__, 'lighting': light})
variants.append(v)

# 2. 驾驶员状态变体
states = ["normal", "fatigue", "distraction", "alcohol"]
for state in states:
if state != base.driver_state:
v = CabinSceneConfig(**{**base.__dict__, 'driver_state': state})
variants.append(v)

# 3. 乘员组合变体
# 只有驾驶员
v1 = CabinSceneConfig(**{**base.__dict__})
v1.passenger_front = False
v1.passenger_rear_left = False
v1.passenger_rear_right = False
variants.append(v1)

# 驾驶员 + 前排乘客
v2 = CabinSceneConfig(**{**base.__dict__})
v2.passenger_front = True
variants.append(v2)

# 全座
v3 = CabinSceneConfig(**{**base.__dict__})
v3.passenger_front = True
v3.passenger_rear_left = True
v3.passenger_rear_right = True
variants.append(v3)

# 有儿童座椅
v4 = CabinSceneConfig(**{**base.__dict__})
v4.child_seat = True
v4.passenger_rear_left = True
variants.append(v4)

# 4. 传感器位置变体
positions = ["dashboard", "overhead", "A-pillar"]
for pos in positions:
if pos != base.camera_position:
v = CabinSceneConfig(**{**base.__dict__, 'camera_position': pos})
variants.append(v)

# 取前 num_variants 个
np.random.shuffle(variants)
self.generated_count += len(variants[:num_variants])
return variants[:num_variants]

def generate_corner_cases(self) -> List[CabinSceneConfig]:
"""生成 corner case 场景"""
corner_cases = [
# 驾驶员突发疾病
CabinSceneConfig(
driver_state="fatigue",
lighting="night",
vehicle_speed=100,
road_type="highway"
),
# 儿童单独在后排(CPD场景)
CabinSceneConfig(
driver_present=False,
child_seat=True,
passenger_rear_left=True,
lighting="day",
vehicle_speed=0
),
# 酒驾 + 夜间
CabinSceneConfig(
driver_state="alcohol",
lighting="night",
vehicle_speed=60,
road_type="urban"
),
# 隧道中分心
CabinSceneConfig(
driver_state="distraction",
lighting="tunnel",
vehicle_speed=80,
road_type="highway"
),
# 儿童在后备箱(极端case)
CabinSceneConfig(
driver_present=False,
child_seat=False,
lighting="day",
vehicle_speed=0,
camera_position="overhead"
),
]
return corner_cases


# ========== 测试 ==========

if __name__ == "__main__":
generator = CosmosCabinGenerator()

# 基础场景:白天高速,驾驶员正常
base = CabinSceneConfig(
driver_present=True,
driver_state="normal",
lighting="day",
vehicle_speed=100,
road_type="highway",
camera_position="dashboard"
)
generator.add_base_scene(base)

# 生成变体
variants = generator.generate_variants(base, num_variants=10)
print(f"=== Cosmos 场景生成 ===")
print(f"基础场景: 1 个")
print(f"生成变体: {len(variants)} 个")
for i, v in enumerate(variants):
print(f" 变体{i+1}: "
f"光照={v.lighting}, "
f"状态={v.driver_state}, "
f"前排乘客={v.passenger_front}, "
f"儿童座椅={v.child_seat}")

# 生成 corner cases
corners = generator.generate_corner_cases()
print(f"\nCorner Cases: {len(corners)} 个")
for i, c in enumerate(corners):
print(f" CC{i+1}: "
f"驾驶员={c.driver_present}, "
f"状态={c.driver_state}, "
f"光照={c.lighting}, "
f"儿童座椅={c.child_seat}")

NVIDIA Robotaxi 技术栈全景

DRIVE Hyperion 10 参考架构

组件 规格
计算 双 DRIVE AGX Thor
摄像头 14 个
雷达 9 个
激光雷达 3 个
超声波 12 个
冗余 计算和传感双重冗余
安全框架 NVIDIA Halos

合作伙伴生态

NVIDIA 列举的 robotaxi 合作伙伴覆盖全产业链:

类别 合作方
Robotaxi平台 Uber, Zoox, Pony.ai, Wayve, Waabi
OEM Mercedes-Benz, Stellantis, Nissan, Lucid, Hyundai, Kia, Geely, Zeekr, Tesla
AD开发商 Momenta, Nuro, May Mobility, Avride
域控制器 Lenovo (AD1 L4), Tensor (8x DRIVE AGX Thor)

推理模型改进

NVIDIA 报告了关键数据:

“Adding meta-action and chain-of-thought reasoning data improved trajectory prediction accuracy and reduced minimum average displacement error by 43%, from 2.08 to 1.18.”

对 IMS 开发的启示

1. NuRec 用于座舱数据合成

NuRec 的核心价值是从真实数据出发重建场景,而非从零建模:

1
真实 DMS 数据 → NuRec 重建 → 编辑视角/光照/乘员 → 生成新训练数据

IMS 数据合成路线:

1
2
3
4
5
6
7
8
9
# IMS NuRec 数据合成管道
pipeline = [
"1. 采集真实座舱多视角数据(5-10个摄像头)",
"2. NuRec 神经重建座舱 3D 场景",
"3. 编辑:改变乘员/姿态/光照/遮挡",
"4. 渲染新视角的 DMS/OMS 图像",
"5. 自动标注(3D->2D投影)",
"6. 生成训练数据集",
]

2. Cosmos 用于 corner case 生成

Cosmos 世界模型可以生成稀有场景:

  • 酒驾 + 夜间 + 雨天
  • 儿童被毯子遮挡 + 热天
  • 突发疾病 + 隧道
  • 多乘员复杂交互

3. 数据工厂架构

NVIDIA 提到的 “AI data factory” 概念:

Uber 正在建设使用 Cosmos 的 robotaxi AI 数据工厂

IMS 数据工厂建议:

阶段 工具 产出
采集 真实车队/测试车 100小时原始数据
重建 NuRec 10个座舱3D场景
变体生成 Cosmos 10000+合成场景
标注 自动3D->2D 100000+标注图像
验证 闭环仿真 模型精度验证
部署 TensorRT 量化 边缘推理

4. 量产芯片选型

芯片 场景 算力 参考
DRIVE AGX Thor 车载 L4 ~2000 TOPS NVIDIA robotaxi 参考
Qualcomm QCS8255 DMS/OMS 26 TOPS IMS 主力芯片
NVIDIA RTX PRO 数据中心仿真 - NuRec/Cosmos 运行

结论

NVIDIA Omniverse NuRec + Cosmos 为 IMS 合成数据提供了完整的工具链:从真实数据重建场景,生成无限变体,自动标注,闭环验证。这比纯手工建模或 GAN 生成更高效、更真实。

对 IMS 开发而言,评估 NuRec 用于座舱场景重建的可行性,并设计”数据工厂”管道,是突破数据瓶颈的关键方向。


参考链接: