1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225
| import torch import torch.nn as nn import torch.nn.functional as F
class SpatialEncoder(nn.Module): """空间编码器:从单帧提取 AU 空间分布""" def __init__(self, n_aus=12): super().__init__() self.backbone = nn.Sequential( nn.Conv2d(3, 32, 3, stride=2, padding=1), nn.BatchNorm2d(32), nn.ReLU(), nn.Conv2d(32, 64, 3, stride=2, padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.Conv2d(64, 128, 3, stride=2, padding=1), nn.BatchNorm2d(128), nn.ReLU(), nn.AdaptiveAvgPool2d((4, 4)), ) self.au_head = nn.Sequential( nn.Linear(128 * 16, 256), nn.ReLU(), nn.Dropout(0.3), nn.Linear(256, n_aus), nn.Sigmoid() ) def forward(self, x): """ Args: x: (B, C, H, W) 单帧面部图像 Returns: aus: (B, n_aus) AU 强度 feat: (B, 2048) 特征向量 """ feat = self.backbone(x) feat_flat = feat.flatten(1) aus = self.au_head(feat_flat) return aus, feat_flat
class TemporalEncoder(nn.Module): """时间编码器:建模 AU 序列动态""" def __init__(self, n_aus=12, hidden_dim=128, n_layers=2): super().__init__() self.gru = nn.GRU( input_size=n_aus, hidden_dim=hidden_dim, num_layers=n_layers, batch_first=True, bidirectional=True ) self.temporal_head = nn.Sequential( nn.Linear(hidden_dim * 2, n_aus), nn.Sigmoid() ) def forward(self, au_sequence): """ Args: au_sequence: (B, T, n_aus) AU 时间序列 Returns: au_pred: (B, T, n_aus) 预测的下一帧 AU hidden: (B, hidden_dim*2) 时序特征 """ out, hidden = self.gru(au_sequence) au_pred = self.temporal_head(out) return au_pred, hidden
class TwinCycleAutoencoder(nn.Module): """ 双循环自编码器 空间循环:图像 → AU → 重建图像 时间循环:AU序列 → 预测下一帧 → 重建序列 两个循环互相约束: - 空间编码的 AU 必须能重建面部图像 - 时间编码的 AU 必须能预测未来 AU """ def __init__(self, n_aus=12): super().__init__() self.spatial = SpatialEncoder(n_aus=n_aus) self.temporal = TemporalEncoder(n_aus=n_aus) self.image_decoder = nn.Sequential( nn.Linear(n_aus, 256), nn.ReLU(), nn.Linear(256, 128 * 4 * 4), nn.ReLU(), ) self.deconv = nn.Sequential( nn.ConvTranspose2d(128, 64, 3, stride=2, padding=1, output_padding=1), nn.BatchNorm2d(64), nn.ReLU(), nn.ConvTranspose2d(64, 32, 3, stride=2, padding=1, output_padding=1), nn.BatchNorm2d(32), nn.ReLU(), nn.ConvTranspose2d(32, 3, 3, stride=2, padding=1, output_padding=1), nn.Sigmoid() ) def forward(self, image_sequence): """ Args: image_sequence: (B, T, C, H, W) 面部视频序列 Returns: dict: 空间 AU、时序 AU、重建图像、预测 AU """ B, T, C, H, W = image_sequence.shape frames = image_sequence.reshape(B*T, C, H, W) au_spatial, _ = self.spatial(frames) au_spatial = au_spatial.reshape(B, T, -1) au_temporal, hidden = self.temporal(au_spatial) au_flat = au_spatial.reshape(B*T, -1) recon_feat = self.image_decoder(au_flat) recon_feat = recon_feat.reshape(B*T, 128, 4, 4) recon_images = self.deconv(recon_feat) recon_images = recon_images.reshape(B, T, C, H, W) return { 'au_spatial': au_spatial, 'au_temporal': au_temporal, 'recon_images': recon_images, 'hidden': hidden, } def compute_loss(self, outputs, targets): """ 计算双循环一致性损失 Args: outputs: forward() 返回值 targets: { 'images': (B, T, C, H, W) 原始图像, 'au_labels': (B, T, n_aus) AU 标签(可选) } """ recon_loss = F.mse_loss( outputs['recon_images'], targets['images'] ) temporal_loss = F.l1_loss( outputs['au_temporal'][:, :-1], outputs['au_spatial'][:, 1:].detach() ) if 'au_labels' in targets: detection_loss = F.binary_cross_entropy( outputs['au_spatial'], targets['au_labels'] ) else: detection_loss = torch.tensor(0.0) total_loss = ( 1.0 * recon_loss + 0.5 * temporal_loss + 1.0 * detection_loss ) return { 'total': total_loss, 'recon': recon_loss, 'temporal': temporal_loss, 'detection': detection_loss }
if __name__ == "__main__": model = TwinCycleAutoencoder(n_aus=12) x = torch.randn(2, 16, 3, 64, 64) x = torch.sigmoid(x) outputs = model(x) print("Twin Cycle Autoencoder 输出:") print(f" AU 空间: {outputs['au_spatial'].shape}") print(f" AU 时序: {outputs['au_temporal'].shape}") print(f" 重建图像: {outputs['recon_images'].shape}") print(f" 隐藏特征: {outputs['hidden'].shape}") targets = {'images': x} losses = model.compute_loss(outputs, targets) print(f"\n损失:") for k, v in losses.items(): print(f" {k}: {v.item():.4f}") print(f"\n参数总数: {sum(p.numel() for p in model.parameters()):,}")
|