DistillGaze:视觉基础模型蒸馏→256K边缘眼动追踪——无需真实gaze标签

论文信息

项目 内容
标题 Rapidly Deploying On-Device Eye Tracking by Distilling Visual Foundation Models
会议 ECCV 2026 (Poster)
链接 ECCV Poster 5564
关键词 知识蒸馏, 眼动追踪, 视觉基础模型, 合成→真实, 边缘部署
核心突破 256K参数模型, 无需真实gaze标签, 误差1.44°

核心发现

两阶段蒸馏:DINOv3(86M)→域优化教师(86M)→边缘学生(256K),全流程无需真实gaze标签,误差从3.48°降到1.44°。

关键指标

指标 基线 DistillGaze 提升
中位gaze误差 3.48° 1.44° -58.6%
尾部误差(E90U90) 14.84° 8.45° -43.1%
模型大小 86M (DINOv3) 256K 336×压缩
真实gaze标签 需要 不需要 零标注
训练数据 真实标注 合成+未标注真实 被动采集

与GazeDiT的关系

方面 GazeDiT (博客#01) DistillGaze (本文)
目标 生成合成眼部图像 蒸馏到边缘设备
方法 扩散模型+物理渲染 VFM自蒸馏+跨架构蒸馏
模型大小 扩散模型(大) 256K(极小)
需要真实gaze标签 不需要 不需要
互补性 生成训练数据 部署到边缘

方法详解

两阶段架构

核心代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
import torch
import torch.nn as nn
import torch.nn.functional as F

class DistillGaze(nn.Module):
"""
DistillGaze: 两阶段蒸馏眼动追踪

Stage 1: DINOv3 → 域优化教师 (自蒸馏)
Stage 2: 教师 → 256K学生 (跨架构蒸馏)

关键创新:
1. 设备感知增强(MTF/运动模糊/眩光)
2. 混合监督VFM优化
3. 特征VIC-KD + 预测伪标签
"""

def __init__(self):
super().__init__()

# === Stage 1: VFM教师 ===
self.teacher = DINOv3Encoder(freeze=False)

# EMA教师(用于自蒸馏一致性)
self.ema_teacher = DINOv3Encoder(freeze=True)
self._update_ema()

# === Stage 2: 边缘学生 ===
self.student = FBNetStudent(model_size='256K')

def _update_ema(self, decay=0.999):
"""EMA参数更新"""
with torch.no_grad():
for ema_p, p in zip(self.ema_teacher.parameters(),
self.teacher.parameters()):
ema_p.mul_(decay).add_(p * (1 - decay))

def stage1_forward(self, synthetic_img, synthetic_label,
real_img_weak, real_img_strong):
"""
Stage 1: VFM域优化

损失 = 合成监督 + 自蒸馏一致性
"""
# 合成图像监督
syn_feat = self.teacher(synthetic_img)
syn_pred = self.gaze_head(syn_feat)
sup_loss = F.mse_loss(syn_pred, synthetic_label)

# 真实图像自蒸馏
with torch.no_grad():
real_feat_weak = self.ema_teacher(real_img_weak)
real_pseudo = self.gaze_head(real_feat_weak)

real_feat_strong = self.teacher(real_img_strong)
real_pred = self.gaze_head(real_feat_strong)

# 一致性损失
cons_loss = F.mse_loss(real_pred, real_pseudo.detach())

total_loss = sup_loss + 2.0 * cons_loss
return total_loss

def stage2_forward(self, img_weak, img_strong):
"""
Stage 2: 边缘蒸馏

教师(86M) → 学生(256K)
损失 = 特征VIC-KD + 预测伪标签
"""
with torch.no_grad():
teacher_feat = self.teacher(img_weak)
teacher_pred = self.gaze_head(teacher_feat)

student_feat = self.student(img_strong)
student_pred = self.gaze_head_student(student_feat)

# 特征蒸馏 (VIC-KD)
feat_loss = self.vic_kd_loss(student_feat, teacher_feat)

# 预测蒸馏
pred_loss = F.mse_loss(student_pred, teacher_pred.detach())

total_loss = feat_loss + pred_loss
return total_loss

def vic_kd_loss(self, student_feat, teacher_feat):
"""VIC-KD: 跨架构特征蒸馏"""
# 投影到公共空间
s_proj = self.student_proj(student_feat)
t_proj = self.teacher_proj(teacher_feat.detach())

# 余弦相似度损失
cos_loss = 1 - F.cosine_similarity(s_proj, t_proj, dim=-1).mean()
return cos_loss


class DINOv3Encoder(nn.Module):
"""DINOv3 ViT-B 编码器 (86M参数)"""
def __init__(self, freeze=True):
super().__init__()
self.vit = nn.Sequential(
nn.Conv2d(3, 768, kernel_size=16, stride=16),
*self._make_vit_layers(n_layers=12, d_model=768, n_heads=12)
)
if freeze:
for p in self.parameters():
p.requires_grad = False

def _make_vit_layers(self, n_layers, d_model, n_heads):
layers = []
for _ in range(n_layers):
layers.append(ViTBlock(d_model, n_heads))
return layers

def forward(self, x):
return self.vit(x)


class FBNetStudent(nn.Module):
"""FBNet 轻量化学生模型 (256K参数)"""
def __init__(self, model_size='256K'):
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(3, 16, 3, stride=2, padding=1),
nn.BatchNorm2d(16), nn.ReLU6(),

nn.Conv2d(16, 32, 3, stride=2, padding=1),
nn.BatchNorm2d(32), nn.ReLU6(),

nn.Conv2d(32, 48, 3, stride=2, padding=1),
nn.BatchNorm2d(48), nn.ReLU6(),

nn.Conv2d(48, 64, 3, stride=1, padding=1),
nn.BatchNorm2d(64), nn.ReLU6(),

nn.AdaptiveAvgPool2d(1),
)
self.proj = nn.Linear(64, 64)

def forward(self, x):
return self.proj(self.features(x).flatten(1))


class ViTBlock(nn.Module):
"""简化版ViT Block"""
def __init__(self, d_model, n_heads):
super().__init__()
self.norm1 = nn.LayerNorm(d_model)
self.attn = nn.MultiheadAttention(d_model, n_heads, batch_first=True)
self.norm2 = nn.LayerNorm(d_model)
self.ffn = nn.Sequential(
nn.Linear(d_model, d_model*4),
nn.GELU(),
nn.Linear(d_model*4, d_model)
)

def forward(self, x):
B, C, H, W = x.shape
x = x.flatten(2).transpose(1, 2) # (B, HW, C)
x = x + self.attn(self.norm1(x), self.norm1(x), self.norm1(x))[0]
x = x + self.ffn(self.norm2(x))
return x.transpose(1, 2).view(B, C, H, W)


# === 设备感知增强 ===
class DeviceAwareAugmentation:
"""设备感知数据增强"""

def __init__(self, strong=True):
self.strong = strong
self.p = 0.3 # 触发概率

def __call__(self, img):
if self.strong:
# 强增强
img = self._apply_camera_aug(img)
img = self._apply_illumination_aug(img)
else:
# 弱增强
img = self._gamma_jitter(img)
img = self._random_scale(img)
return img

def _apply_camera_aug(self, img):
"""相机退化模拟"""
if torch.rand(1) < self.p:
# MTF滤波(调制传递函数)
img = self._mtf_filter(img)
if torch.rand(1) < self.p:
# 运动模糊
img = self._motion_blur(img)
if torch.rand(1) < self.p:
# 压缩伪影
img = self._compression(img)
return img

def _apply_illumination_aug(self, img):
"""照明退化"""
if torch.rand(1) < self.p:
# 角膜眩光
img = self._glint_inpaint(img)
if torch.rand(1) < self.p:
# 亮度/对比度
img = self._brightness_contrast(img)
return img

def _gamma_jitter(self, img):
gamma = torch.uniform(0.8, 1.2, (1,)).item()
return img ** gamma

def _random_scale(self, img):
s = torch.uniform(0.9, 1.1, (1,)).item()
return F.interpolate(img.unsqueeze(0), scale_factor=s).squeeze(0)

def _mtf_filter(self, img):
return img # 简化

def _motion_blur(self, img):
return img # 简化

def _compression(self, img):
return img # 简化

def _glint_inpaint(self, img):
return img # 简化

def _brightness_contrast(self, img):
return img * torch.uniform(0.8, 1.2, (1,)).item()


# 测试
if __name__ == "__main__":
model = DistillGaze()

# 模拟输入
B = 4
syn_img = torch.randn(B, 3, 64, 64)
syn_label = torch.randn(B, 2) # [yaw, pitch]
real_weak = torch.randn(B, 3, 64, 64)
real_strong = torch.randn(B, 3, 64, 64)

# Stage 1
loss1 = model.stage1_forward(syn_img, syn_label, real_weak, real_strong)
print(f"Stage 1 损失: {loss1.item():.4f}")

# Stage 2
loss2 = model.stage2_forward(real_weak, real_strong)
print(f"Stage 2 损失: {loss2.item():.4f}")

# 学生模型大小
student_params = sum(p.numel() for p in model.student.parameters())
teacher_params = sum(p.numel() for p in model.teacher.parameters())
print(f"\n教师参数量: {teacher_params/1e6:.1f}M")
print(f"学生参数量: {student_params/1e3:.1f}K")
print(f"压缩比: {teacher_params/student_params:.0f}x")

# 增强测试
aug = DeviceAwareAugmentation(strong=True)
augmented = aug(syn_img[0])
print(f"\n增强后形状: {augmented.shape}")

IMS 开发启示

新硬件适配流程

传统流程 vs DistillGaze流程:

步骤 传统流程 DistillGaze 时间
1 硬件原型 硬件原型 1-2周
2 标定+数据采集 合成数据渲染 数月→数天
3 人工gaze标注 被动采集未标注图像 省去标注
4 训练模型 两阶段蒸馏 1-2周
5 部署 256K模型直接部署 即时
总计 3-6月 2-3周 10x加速

256K模型意味着什么?

对比项 传统DMS模型 DistillGaze
模型大小 5-20 MB 256KB
推理延迟 5-20ms <1ms
NPU占用 5-10% <0.1%
内存占用 50-200MB <1MB
功耗 100-500mW <10mW

增强策略的价值

DistillGaze 的设备感知增强直接针对DMS痛点:

增强类型 DMS对应问题
MTF滤波 摄像头分辨率不足
运动模糊 车辆振动
压缩伪影 视频编码损失
角膜眩光 IR补光反射
随机阴影 光照变化

与GazeDiT组合管道

Euro NCAP 关联

Euro NCAP场景 DistillGaze价值
D-01 视线偏离 1.44°精度远超NCAP要求
D-02/D-03 手机使用 高精度gaze+设备感知增强
F-01~F-05 疲劳检测 低延迟<1ms支持实时PERCLOS
C-01~C-03 认知分心 高精度gaze可检测细微偏离

局限性

  1. Project Aria基准 — 测试在AR/VR场景,座舱场景未验证
  2. 红外图像 — 论文使用NIR图像,座舱也是IR但几何不同
  3. 单人假设 — 仅单驾驶员,未测试多乘员
  4. 256K模型容量 — 可能无法处理极端情况
  5. 教师依赖 — Stage 2质量取决于Stage 1教师

总结

DistillGaze 是眼动追踪部署的突破性工作。对 IMS 团队:

  1. 256K模型可在QCS8255上<1ms推理 — 几乎不占资源
  2. 无需真实gaze标签 — 新硬件适配从3月→2周
  3. 1.44°精度满足Euro NCAP — 远超D-01场景要求
  4. 与GazeDiT组合 — 合成生成+蒸馏部署的完整管道
  5. 设备感知增强直接适用座舱 — 振动/眩光/压缩全覆盖
  6. DINOv3→领域教师的思路可迁移 — 其他DMS任务也可用VFM蒸馏

参考链接: