LISA：语言引导的驾驶员视线估计新方法

论文信息

• 标题： LISA: Language-guided Interference-aware Spatial-Frequency Attention for Driver Gaze Estimation
• 作者： Jinpeng Chen, Jun Ma等
• 发表时间： 2026年5月
• 来源： arXiv:2605.17287
• 核心创新： 语言引导+空间-频率注意力，遮挡和光照鲁棒性显著提升

核心创新

LISA提出了一种语言引导的干扰感知空间-频率注意力机制，解决传统视线估计的三大挑战：

1. 遮挡鲁棒性：墨镜、眼镜遮挡下仍能准确估计
2. 光照适应性：强光、暗光、隧道穿越等极端条件
3. 跨域泛化：不同车型、不同驾驶员泛化能力强

方法详解

1. 系统架构

flowchart TD
    A[RGB图像输入] --> B[视觉编码器 ViT]
    B --> C[空间注意力模块]
    B --> D[频率注意力模块]
    
    E[文本描述] --> F[语言编码器 BERT]
    F --> G[跨模态对齐]
    
    C --> H[特征融合]
    D --> H
    G --> H
    
    H --> I[干扰抑制模块]
    I --> J[视线分类器]
    
    subgraph 双流编码
        B
        F
    end
    
    subgraph 注意力融合
        C
        D
        G
        H
    end

2. 空间-频率双域注意力

import torch
import torch.nn as nn
import torch.nn.functional as F

class SpatialFrequencyAttention(nn.Module):
    """
    空间-频率双域注意力
    
    同时建模空间依赖和频率特征
    """
    
    def __init__(self, dim: int = 256, num_heads: int = 8):
        super().__init__()
        self.dim = dim
        self.num_heads = num_heads
        self.head_dim = dim // num_heads
        
        # 空间注意力
        self.spatial_attn = nn.MultiheadAttention(dim, num_heads)
        
        # 频率注意力
        self.freq_attn = nn.MultiheadAttention(dim, num_heads)
        
        # 融合层
        self.fusion = nn.Linear(dim * 2, dim)
        
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Args:
            x: 输入特征, shape=(B, N, C)
               B: batch size
               N: 序列长度（patch数）
               C: 特征维度
        
        Returns:
            out: 增强特征, shape=(B, N, C)
        """
        B, N, C = x.shape
        
        # 空间注意力
        spatial_out, _ = self.spatial_attn(x, x, x)
        
        # FFT转频域
        x_freq = torch.fft.fft2(x.float())
        x_freq_real = torch.real(x_freq)
        x_freq_imag = torch.imag(x_freq)
        x_freq_concat = torch.cat([x_freq_real, x_freq_imag], dim=-1)
        
        # 频率注意力（简化）
        freq_out, _ = self.freq_attn(
            x_freq_concat[..., :C],
            x_freq_concat[..., :C],
            x_freq_concat[..., :C]
        )
        
        # 融合
        concat = torch.cat([spatial_out, freq_out], dim=-1)
        out = self.fusion(concat)
        
        return out


class LISAModel(nn.Module):
    """
    LISA完整模型
    
    语言引导的视线估计
    """
    
    def __init__(
        self,
        vision_dim: int = 768,
        text_dim: int = 768,
        hidden_dim: int = 256,
        num_gaze_zones: int = 9
    ):
        super().__init__()
        
        # 视觉编码器（预训练ViT）
        self.vision_encoder = VisionTransformer()
        
        # 文本编码器（预训练BERT）
        self.text_encoder = TextTransformer()
        
        # 双域注意力
        self.dual_attn = SpatialFrequencyAttention(hidden_dim)
        
        # 跨模态对齐
        self.cross_modal = CrossModalAlignment(vision_dim, text_dim, hidden_dim)
        
        # 干扰抑制
        self.interference_suppressor = InterferenceSuppressor(hidden_dim)
        
        # 分类器
        self.classifier = nn.Linear(hidden_dim, num_gaze_zones)
    
    def forward(
        self,
        image: torch.Tensor,
        text: torch.Tensor
    ) -> torch.Tensor:
        """
        Args:
            image: 图像, shape=(B, 3, H, W)
            text: 文本token, shape=(B, L)
        
        Returns:
            logits: 视线区域分类, shape=(B, num_gaze_zones)
        """
        # 编码
        vision_feat = self.vision_encoder(image)  # (B, N, C)
        text_feat = self.text_encoder(text)       # (B, L, C)
        
        # 跨模态对齐
        aligned_feat = self.cross_modal(vision_feat, text_feat)
        
        # 双域注意力
        enhanced_feat = self.dual_attn(aligned_feat)
        
        # 干扰抑制
        clean_feat = self.interference_suppressor(enhanced_feat)
        
        # 分类
        logits = self.classifier(clean_feat.mean(dim=1))
        
        return logits


class CrossModalAlignment(nn.Module):
    """跨模态对齐"""
    
    def __init__(self, vision_dim: int, text_dim: int, hidden_dim: int):
        super().__init__()
        
        self.vision_proj = nn.Linear(vision_dim, hidden_dim)
        self.text_proj = nn.Linear(text_dim, hidden_dim)
        
    def forward(self, vision_feat, text_feat):
        # 投影到共享空间
        v = self.vision_proj(vision_feat)
        t = self.text_proj(text_feat)
        
        # 跨模态注意力
        # 视觉特征attend to文本特征
        attn = torch.matmul(v, t.transpose(-1, -2))
        attn = F.softmax(attn / (v.shape[-1] ** 0.5), dim=-1)
        
        aligned = torch.matmul(attn, t)
        
        return v + aligned


class InterferenceSuppressor(nn.Module):
    """
    干扰抑制模块
    
    抑制遮挡和光照干扰
    """
    
    def __init__(self, dim: int):
        super().__init__()
        
        self.gate = nn.Sequential(
            nn.Linear(dim, dim),
            nn.Sigmoid()
        )
        
    def forward(self, x):
        # 学习干扰掩码
        gate = self.gate(x)
        
        # 抑制干扰区域
        clean = x * gate
        
        return clean


# 简化版视觉编码器
class VisionTransformer(nn.Module):
    """简化版ViT"""
    
    def __init__(self, dim: int = 768):
        super().__init__()
        self.conv = nn.Conv2d(3, dim, kernel_size=16, stride=16)
        self.pos_embed = nn.Parameter(torch.randn(1, 197, dim))
        
    def forward(self, x):
        B = x.shape[0]
        x = self.conv(x).flatten(2).transpose(1, 2)
        x = x + self.pos_embed[:, :x.shape[1]]
        return x


class TextTransformer(nn.Module):
    """简化版BERT"""
    
    def __init__(self, dim: int = 768):
        super().__init__()
        self.embed = nn.Embedding(30522, dim)  # BERT词表
        
    def forward(self, x):
        return self.embed(x)


# 示例
if __name__ == "__main__":
    model = LISAModel()
    
    # 模拟输入
    image = torch.randn(2, 3, 224, 224)
    text = torch.randint(0, 30522, (2, 20))
    
    # 前向传播
    logits = model(image, text)
    
    print(f"输入图像: {image.shape}")
    print(f"输入文本: {text.shape}")
    print(f"输出logits: {logits.shape}")
    print(f"预测视线区域: {logits.argmax(dim=-1)}")

3. 文本描述模板

# 视线估计任务的文本描述模板
GAZE_DESCRIPTIONS = {
    "normal": "driver looking at road ahead normally",
    "left_mirror": "driver checking left side mirror",
    "right_mirror": "driver checking right side mirror",
    "dashboard": "driver looking at dashboard",
    "infotainment": "driver interacting with infotainment screen",
    "passenger": "driver turning head to passenger side",
    "phone": "driver looking down at phone",
    "radio": "driver adjusting radio controls",
    "occluded": "driver face partially occluded by sunglasses"
}


def encode_gaze_descriptions(encoder: TextTransformer):
    """
    编码所有视线区域描述
    
    Returns:
        descriptions: (num_zones, L, dim)
    """
    descriptions = []
    
    for zone, desc in GAZE_DESCRIPTIONS.items():
        # 简化：直接tokenize（实际需要BERT tokenizer）
        tokens = torch.randint(0, 30522, (1, 20))
        encoded = encoder(tokens)
        descriptions.append(encoded)
    
    return torch.cat(descriptions, dim=0)


# 示例
if __name__ == "__main__":
    encoder = TextTransformer()
    desc_features = encode_gaze_descriptions(encoder)
    
    print(f"视线区域数: {desc_features.shape[0]}")
    print(f"文本特征维度: {desc_features.shape}")

实验结果

性能对比

方法	正常光照	强光	暗光	遮挡	平均
ResNet-50	92.3%	75.6%	68.4%	60.2%	74.1%
ViT-Base	93.5%	80.2%	72.1%	65.8%	77.9%
LISA	95.2%	89.7%	85.3%	82.1%	88.1%

遮挡场景分析

遮挡类型	传统方法	LISA	改进
无遮挡	92.3%	95.2%	+2.9%
墨镜	58.3%	82.1%	+23.8%
眼镜	70.5%	88.6%	+18.1%
部分遮挡	65.8%	85.4%	+19.6%

IMS开发启示

1. 模型部署

# gaze-config.yaml
gaze_estimation:
  model:
    name: "LISA"
    backbone: "ViT-Base"
    pretrained: "imagenet"
    
  input:
    image_size: [224, 224]
    fps: 15
    color_space: "RGB"
    
  output:
    num_zones: 9
    zones:
      - "road_ahead"
      - "left_mirror"
      - "right_mirror"
      - "dashboard"
      - "infotainment"
      - "passenger"
      - "phone"
      - "radio"
      - "unknown"
    
  optimization:
    quantization: "int8"
    latency_budget: 20  # ms
    
  deployment:
    platform: "QCS8255"
    accelerator: "Hexagon NPU"

2. 视线区域定义

# Euro NCAP视线区域定义
class GazeZone:
    """视线区域定义"""
    
    # 分心相关区域
    DISTRACTION_ZONES = [
        "infotainment",  # 中控屏
        "phone",         # 手机
        "passenger"      # 乘客侧
    ]
    
    # 正常区域
    NORMAL_ZONES = [
        "road_ahead",    # 前方道路
        "left_mirror",   # 左后视镜
        "right_mirror"   # 右后视镜
    ]
    
    # 判断逻辑
    @staticmethod
    def is_distracted(zone: str, duration: float) -> bool:
        """
        判断是否分心
        
        Args:
            zone: 视线区域
            duration: 持续时间（秒）
        
        Returns:
            is_distracted: 是否分心
        """
        if zone in GazeZone.DISTRACTION_ZONES:
            # 分心区域持续时间>2秒即报警
            return duration > 2.0
        return False

3. 实现优先级

优先级	模块	工作量	备注
P0	ViT模型部署	2周	ONNX转换
P0	视线分类器	1周	9类分类
P1	文本编码器	1周	BERT-lite
P1	双域注意力	2周	空间+频率
P2	干扰抑制	1周	遮挡鲁棒

结论

LISA为驾驶员视线估计提供了新思路：

1. 遮挡鲁棒：墨镜遮挡下准确率82.1%
2. 光照适应：强光/暗光下准确率保持85%+
3. 跨域泛化：语言引导增强泛化能力

对于IMS开发，建议：

• P0优先部署ViT基础模型
• 逐步集成双域注意力
• 建立完整的分心检测逻辑

---

参考实现： 完整代码已上传GitHub。