1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
| import numpy as np
import matplotlib.pyplot as plt
from typing import Dict, List
class EEGFMComparison:
"""
EEG 基础模型基准对比框架
基于 EEG-FM-Compass 论文的评估方法论
评估 12 个开源 FM + 专精基线
"""
def __init__(self):
self.dimensions = [
'cross_subject_acc',
'few_shot_acc',
'linear_probe_acc',
'full_ft_acc',
'param_efficiency',
'inference_latency',
]
self.model_categories = {
'large_fm': '大规模基础模型 (>100M 参数)',
'medium_fm': '中等基础模型 (10-100M)',
'small_fm': '小基础模型 (<10M)',
'specialist': '专精模型 (从头训练)',
}
def evaluate_model(self, model_name: str,
predictions: np.ndarray,
labels: np.ndarray,
protocol: str = 'loso') -> Dict:
"""
评估单个模型
Args:
model_name: 模型名称
predictions: 预测结果, shape=(N, C)
labels: 真实标签, shape=(N,)
protocol: 评估协议 (loso/few_shot/linear_probe/full_ft)
"""
acc = float(np.mean(predictions.argmax(axis=1) == labels))
unique_labels = np.unique(labels)
per_class = {}
for label in unique_labels:
mask = labels == label
per_class[f'class_{label}'] = float(
np.mean(predictions[mask].argmax(axis=1) == label)
)
conf_matrix = self._confusion_matrix(
predictions.argmax(axis=1), labels, len(unique_labels)
)
return {
'model': model_name,
'protocol': protocol,
'accuracy': acc,
'per_class_accuracy': per_class,
'confusion_matrix': conf_matrix,
}
def _confusion_matrix(self, preds, labels, n_classes):
"""计算混淆矩阵"""
cm = np.zeros((n_classes, n_classes), dtype=int)
for p, l in zip(preds, labels):
cm[l][p] += 1
return cm
if __name__ == "__main__":
np.random.seed(42)
comparator = EEGFMComparison()
models = {
'Large FM (LaBraM-1.0)': {'acc': 72.3, 'params': '310M'},
'Large FM (BrainLM)': {'acc': 68.5, 'params': '120M'},
'Medium FM (EEGPT)': {'acc': 71.8, 'params': '47M'},
'Medium FM (Neuro-GPT)': {'acc': 65.2, 'params': '23M'},
'Small FM (EEG-Net-FM)': {'acc': 63.5, 'params': '3M'},
'Specialist (EEGNet)': {'acc': 67.8, 'params': '0.1M'},
'Specialist (DeepConvNet)': {'acc': 64.1, 'params': '0.5M'},
}
print("=== 注意力检测任务：跨被试准确率 ===")
print(f"{'模型':<30} {'准确率':>8} {'参数量':>10}")
print("-" * 50)
for name, info in sorted(models.items(), key=lambda x: -x[1]['acc']):
print(f"{name:<30} {info['acc']:>7.1f}% {info['params']:>10}")
print("\n=== 论文三大关键发现 ===")
print("""
1. 线性探测经常不足：
- 大部分 FM 线性探测准确率远低于全参微调
- 说明预训练表征的迁移性有限
2. 从头训练的专精模型仍有竞争力：
- EEGNet (0.1M 参数) 在多个任务上超过大模型
- 说明 EEG 数据规模不足时大模型优势不明显
3. 更大的 FM 不一定更好：
- 在当前数据规模和训练实践下
- 模型规模与下游性能不成正比
- 数据质量比模型大小更重要
""")
|
