1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
| """
论文：Rapidly Deploying On-Device Eye Tracking by Distilling Visual Foundation Models
会议：ECCV 2026
核心方法：两阶段蒸馏 DINOv3 → 256K端侧眼动追踪
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
class Stage1SelfDistillation(nn.Module):
"""
阶段1: DINOv3自蒸馏为域教师
- 冻结DINOv3提供初始特征
- 使用合成数据(有标签) + 真实红外(无标签)
- 输出域专业教师(86M)
"""
def __init__(self, dinov3: nn.Module, embed_dim: int = 768):
super().__init__()
self.dinov3 = dinov3
for p in self.dinov3.parameters():
p.requires_grad = False
self.gaze_head = nn.Sequential(
nn.Linear(embed_dim, 512),
nn.GELU(),
nn.Dropout(0.1),
nn.Linear(512, 256),
nn.GELU(),
nn.Linear(256, 2)
)
self.projector = nn.Sequential(
nn.Linear(embed_dim, 512),
nn.GELU(),
nn.Linear(512, 512)
)
def forward(self, x):
"""提取DINOv3特征"""
with torch.no_grad():
features = self.dinov3(x)
gaze = self.gaze_head(features)
proj = self.projector(features)
return gaze, proj
def compute_loss(self, x_synth, gaze_label_synth, x_real):
"""
损失 = 合成监督 + 真实自蒸馏一致性
"""
gaze_synth, proj_synth = self.forward(x_synth)
loss_supervised = F.mse_loss(gaze_synth, gaze_label_synth)
gaze_real, proj_real = self.forward(x_real)
with torch.no_grad():
pseudo_gaze = gaze_real.clone()
loss_pseudo = F.mse_loss(gaze_real, pseudo_gaze.detach())
total_loss = loss_supervised + 0.5 * loss_pseudo
return total_loss
class Stage2StudentDistillation(nn.Module):
"""
阶段2: 域教师 → 256K端侧学生
学生模型仅256K参数
使用教师软标签 + 合成标签
无真实眼动标签
"""
def __init__(self, teacher: nn.Module):
super().__init__()
self.teacher = teacher
for p in self.teacher.parameters():
p.requires_grad = False
self.student = TinyGazeNet()
def forward(self, x):
return self.student(x)
def compute_loss(self, x, gaze_label_synth=None):
"""
蒸馏损失 = KL散度 + MSE
"""
with torch.no_grad():
teacher_gaze, _ = self.teacher(x)
student_gaze = self.student(x)
loss_distill = F.mse_loss(student_gaze, teacher_gaze)
if gaze_label_synth is not None:
loss_supervised = F.mse_loss(student_gaze, gaze_label_synth)
return loss_distill + 0.5 * loss_supervised
return loss_distill
class TinyGazeNet(nn.Module):
"""
256K参数的微型眼动追踪网络
用于端侧部署
"""
def __init__(self):
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(1, 16, 3, stride=2, padding=1),
nn.BatchNorm2d(16),
nn.ReLU6(),
nn.Conv2d(16, 32, 3, stride=2, padding=1),
nn.BatchNorm2d(32),
nn.ReLU6(),
nn.Conv2d(32, 32, 3, stride=2, padding=1),
nn.BatchNorm2d(32),
nn.ReLU6(),
nn.AdaptiveAvgPool2d(1),
)
self.gaze_regressor = nn.Sequential(
nn.Linear(32, 32),
nn.ReLU6(),
nn.Linear(32, 2)
)
def forward(self, x):
feat = self.features(x).flatten(1)
gaze = self.gaze_regressor(feat)
return gaze
if __name__ == "__main__":
student = TinyGazeNet()
params = sum(p.numel() for p in student.parameters())
print(f"学生模型参数量: {params:,} ({params/1024:.0f}K)")
x = torch.randn(1, 1, 64, 64)
gaze = student(x)
print(f"输入: {x.shape}")
print(f"输出: {gaze.shape} (pitch, yaw)")
print(f"\n=== 论文结果 ===")
print(f"基线(256K): 3.48° 中位误差")
print(f"DINOv3(86M): >5° (直接用反而更差!)")
print(f"DistillGaze(256K): 1.44° 中位误差 (-58.6%)")
print(f"尾部误差: 14.84° → 8.45° (-43.1%)")
print(f"真实标签: 不需要!")
|
