1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
| import torch import torch.nn as nn
class SGAPGaze(nn.Module): """ SGAP-Gaze: Scene Grid Attention Point-of-Gaze Estimation 创新: 将场景图像分成网格,用注意力机制 学习驾驶员注视落在哪个网格区域 输入: 驾驶员面部图 + 场景图 输出: 注视点坐标(x, y) + 网格注意力图 """ def __init__(self, grid_size: int = 8, feat_dim: int = 256): super().__init__() self.grid_size = grid_size self.face_encoder = nn.Sequential( nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.ReLU(), nn.AdaptiveAvgPool2d((1, 1)), nn.Flatten(), nn.Linear(128, feat_dim) ) self.scene_encoder = nn.Sequential( nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.ReLU(), nn.Conv2d(128, 256, 3, padding=1), nn.ReLU(), ) self.grid_attention = nn.MultiheadAttention( embed_dim=feat_dim, num_heads=8, batch_first=True ) self.pog_head = nn.Sequential( nn.Linear(feat_dim * 2, 128), nn.ReLU(), nn.Linear(128, 2) ) def forward(self, face_img: torch.Tensor, scene_img: torch.Tensor) -> dict: """ Args: face_img: (B, 3, 128, 128) 驾驶员面部 scene_img: (B, 3, 256, 256) 前方场景 Returns: pog: (B, 2) 注视点坐标 attention_map: (B, grid*grid) 网格注意力 """ face_feat = self.face_encoder(face_img) scene_feat = self.scene_encoder(scene_img) B, C, H, W = scene_feat.shape grid_feats = scene_feat.view(B, C, H * W).permute(0, 2, 1) query = face_feat.unsqueeze(1) attn_out, attn_weights = self.grid_attention( query, grid_feats, grid_feats ) combined = torch.cat([face_feat, attn_out.squeeze(1)], dim=1) pog = self.pog_head(combined) return { 'pog': pog, 'attention_map': attn_weights.squeeze(1), 'grid_size': (H, W) }
if __name__ == "__main__": model = SGAPGaze(grid_size=8, feat_dim=256) face = torch.randn(4, 3, 128, 128) scene = torch.randn(4, 3, 256, 256) out = model(face, scene) print(f"面部输入: {face.shape}") print(f"场景输入: {scene.shape}") print(f"注视点: {out['pog'].shape} (归一化坐标)") print(f"注意力图: {out['attention_map'].shape}") print(f"网格尺寸: {out['grid_size']}")
|