1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103
| class LanguageGuidedAttention(nn.Module): """ 语言引导注意力模块 用场景描述文本引导视线区域分类 例如:"驾驶员低头看手机" → 关注手机区域 """ SCENE_DESCRIPTIONS = [ "driver looking at road ahead", "driver looking at center console", "driver looking at phone", "driver looking at left mirror", "driver looking at right mirror", "driver looking down at lap", "driver looking at passenger", "driver eyes closed", ] def __init__(self, n_classes=8, text_dim=768): super().__init__() self.text_encoder = self._load_clip() self.cross_attn = nn.MultiheadAttention( embed_dim=512, num_heads=8, kdim=text_dim, vdim=text_dim ) self.classifier = nn.Linear(512, n_classes) def _load_clip(self): """加载预训练 CLIP 文本编码器""" return nn.Identity() def forward(self, visual_feat, text_embed): """ Args: visual_feat: (B, seq, 512) 视觉特征序列 text_embed: (B, text_dim) 场景描述嵌入 """ text = text_embed.unsqueeze(1) out, _ = self.cross_attn( query=visual_feat, key=text, value=text ) pooled = out.mean(dim=1) logits = self.classifier(pooled) return logits
class LISA(nn.Module): """LISA: Language-guided Interference-aware Spatial-Frequency Attention""" def __init__(self, n_gaze_classes=8): super().__init__() self.backbone = nn.Sequential( nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(128, 256, 3, stride=2, padding=1), nn.ReLU(), nn.AdaptiveAvgPool2d((4, 4)), ) self.sfa = SpatialFrequencyAttention(256) self.lang_guide = LanguageGuidedAttention(n_gaze_classes) def forward(self, image, text_embed): feat = self.backbone(image) feat = self.sfa(feat) feat_flat = feat.flatten(2).permute(2, 0, 1) proj = nn.Linear(256, 512).to(feat.device) feat_proj = proj(feat_flat) logits = self.lang_guide(feat_proj, text_embed) return logits
if __name__ == "__main__": model = LISA() image = torch.randn(2, 3, 128, 128) text_embed = torch.randn(2, 768) logits = model(image, text_embed) print(f"输出: {logits.shape}") print(f"参数: {sum(p.numel() for p in model.parameters()):,}")
|