多模态认知负荷分类:fNIRS+ECG+眼动的 IMS 可用性分析

论文1: Wearable Multimodal Physiological Sensor-Based Recognition of Pilots’ Mental Workload Using fNIRS, ECG, and Eye-Movement Signals
期刊: IEEE Transactions on Instrumentation and Measurement (TIM), 2026
作者: Xu et al.
被试: 27名飞行学员
准确率: 85.76% (10-fold CV), 78.57% (LOSO)

论文2: Classifying Mental Workload Using Psychophysiological Measurement and Machine Learning
会议: ICIEA 2026 (12th International Conference on Industrial Engineering and Applications)
准确率: 75% (仅瞳孔), >70% (多模态融合)

链接: https://publications.artinis.com/publication/xu-wearable-2026/

两篇论文的核心发现

论文1 (Xu et al., IEEE TIM 2026)

关键发现1:PFC激活模式颠覆预期

认知负荷 PFC激活 左右差异
Underload (低负荷) 中等 左PFC > 右PFC
Moderate (适中) 最低 左PFC > 右PFC
Overload (过载) 最高 右PFC > 左PFC

反直觉发现: 适中负荷时PFC激活最低(效率最高),过载和低负荷都需要更多认知资源。这直接挑战了”负荷越高激活越高”的线性假设。

关键发现2:最优特征组合

特征 来源 重要性
β振幅 (CH18, CH20) fNIRS 左PFC BA10区 最高
Mean NN ECG/HRV 高
W分量 ECG超低频功率 高
平均瞳孔直径 眼动 高

关键发现3:SHAP可解释性

SHAP分析揭示的前5重要特征:β_CH18 > β_CH20 > Mean_NN > W > pupil_diameter

论文2 (Tang et al., ICIEA 2026)

模态 最佳准确率 分类器
仅瞳孔 75% LDA
fNIRS+ECG+瞳孔 >70% LDA/DT
单模态fNIRS 38-64% 各分类器

IMS 可用性映射

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
"""
多模态认知负荷分类 → IMS可用性映射

论文使用的3种模态:
1. fNIRS (前额叶血氧) → IMS不可用(需头戴设备)
2. ECG/HRV → IMS可用(方向盘电极)
3. 眼动/瞳孔 → IMS可用(DMS摄像头)

关键问题: 去掉fNIRS后, 仅ECG+眼动能达到多少准确率?
"""

ims_modalities = {
"fNIRS": {
"论文中的角色": "最重要特征(β振幅CH18/CH20)",
"IMS可用": False,
"原因": "需头戴fNIRS设备, 无法车载",
"替代方案": "无直接替代(脑氧合信息无法从外部获取)",
"影响": "去掉后准确率预计下降10-15%"
},
"ECG/HRV": {
"论文中的角色": "第二重要(Mean NN, W分量)",
"IMS可用": True,
"传感器": "方向盘电极 + 座椅电极",
"可实现特征": ["Mean NN", "RMSSD", "LF/HF", "W分量"],
"部署难度": "中(运动伪影需滤波)",
"信心": "★★★★☆"
},
"眼动/瞳孔": {
"论文中的角色": "第三重要(平均瞳孔直径)",
"IMS可用": True,
"传感器": "DMS摄像头 (RGB-IR, 全局快门)",
"可实现特征": ["瞳孔直径", "眨眼率", "注视点", "扫视熵"],
"部署难度": "低(现有DMS已有)",
"信心": "★★★★★"
}
}

print("=== IMS多模态认知负荷检测可用性 ===\n")
for modality, info in ims_modalities.items():
print(f"【{modality}】")
print(f" 论文角色: {info['论文中的角色']}")
print(f" IMS可用: {'✓' if info['IMS可用'] else '✗'}")
if info['IMS可用']:
print(f" 传感器: {info['传感器']}")
print(f" 信心: {info['信心']}")
else:
print(f" 替代: {info['替代方案']}")
print(f" 影响: {info['影响']}")
print()

代码复现:ECG+眼动双模态认知负荷分类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
"""
ECG + 眼动双模态认知负荷分类器

基于论文发现:
- Mean NN (ECG) 是第3重要特征
- W分量 (ECG超低频) 是第4重要特征
- 瞳孔直径 是第5重要特征

去掉fNIRS后, 使用ECG+眼动实现3级分类:
- Underload (低负荷)
- Moderate (适中)
- Overload (过载)
"""

import numpy as np
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import cross_val_score, LeaveOneGroupOut
from sklearn.preprocessing import StandardScaler
from scipy.signal import find_peaks, butter, filtfilt, welch
from typing import Tuple

class CognitiveLoadClassifier:
"""ECG+眼动双模态认知负荷分类器"""

def __init__(self):
self.scaler = StandardScaler()
self.model = GradientBoostingClassifier(
n_estimators=150,
max_depth=4,
learning_rate=0.1,
subsample=0.8,
random_state=42
)
# 3级分类: 0=underload, 1=moderate, 2=overload
self.labels = ['低负荷', '适中', '过载']

def extract_ecg_features(self, ecg: np.ndarray, fs=250) -> np.ndarray:
"""提取ECG/HRV特征: Mean NN, RMSSD, LF/HF, W分量"""
peaks, _ = find_peaks(ecg, distance=150, prominence=0.5)
if len(peaks) < 10:
return np.zeros(4)

rr = np.diff(peaks) / fs
mean_nn = np.mean(rr)
rmssd = np.sqrt(np.mean(np.diff(rr)**2))

freqs, psd = welch(rr, fs=1/mean_nn, nperseg=min(len(rr), 64))
lf = np.sum(psd[(freqs >= 0.04) & (freqs < 0.15)])
hf = np.sum(psd[(freqs >= 0.15) & (freqs < 0.40)])
w = np.sum(psd[(freqs >= 0.01) & (freqs < 0.04)])

return np.array([mean_nn, rmssd, lf/(hf+1e-8), w])

def extract_eye_features(self, eye_data: np.ndarray, fs=30) -> np.ndarray:
"""提取眼动特征: 瞳孔直径, 眨眼率, 注视稳定性"""
eye_openness, gaze_x, gaze_y, pupil_d = eye_data.T

# 瞳孔直径均值和方差
pupil_mean = np.mean(pupil_d)
pupil_var = np.std(pupil_d)

# 眨眼率
blink_threshold = 0.3
blink_events = np.diff((eye_openness < blink_threshold).astype(int))
blink_rate = np.sum(blink_events == -1) / (len(eye_openness) / fs / 60)

# 注视稳定性(注视点分布的标准差)
gaze_std = np.std(np.sqrt(gaze_x**2 + gaze_y**2))

return np.array([pupil_mean, pupil_var, blink_rate, gaze_std])

def extract_features(self, ecg, eye_data):
"""提取双模态特征"""
ecg_f = self.extract_ecg_features(ecg)
eye_f = self.extract_eye_features(eye_data)
return np.concatenate([ecg_f, eye_f])

def train(self, X, y, groups=None):
"""训练模型"""
X_scaled = self.scaler.fit_transform(X)

if groups is not None:
# LOSO交叉验证
logo = LeaveOneGroupOut()
scores = cross_val_score(
self.model, X_scaled, y,
groups=groups, cv=logo, scoring='accuracy'
)
print(f"LOSO准确率: {scores.mean():.4f} ± {scores.std():.4f}")

self.model.fit(X_scaled, y)
return self.model

def predict(self, ecg, eye_data):
"""预测认知负荷等级"""
features = self.extract_features(ecg, eye_data).reshape(1, -1)
features_scaled = self.scaler.transform(features)
pred = self.model.predict(features_scaled)[0]
proba = self.model.predict_proba(features_scaled)[0]

return {
'level': int(pred),
'level_name': self.labels[int(pred)],
'confidence': float(max(proba)),
'probabilities': {self.labels[i]: float(p) for i, p in enumerate(proba)}
}


# ========== 模拟测试 ==========
if __name__ == "__main__":
np.random.seed(42)
clf = CognitiveLoadClassifier()

# 生成模拟训练数据(3个负荷等级, 每级30样本)
n_per_class = 30
X_list, y_list = [], []

for level in range(3):
for _ in range(n_per_class):
# ECG特征: 随负荷变化
mean_nn = 0.9 - level * 0.1 + np.random.normal(0, 0.05)
rmssd = 0.05 - level * 0.01 + np.random.normal(0, 0.01)
lf_hf = 1.0 + level * 0.5 + np.random.normal(0, 0.2)
w = 0.005 - level * 0.001 + np.random.normal(0, 0.002)

# 眼动特征: 随负荷变化
pupil = 3.5 + level * 0.3 + np.random.normal(0, 0.2)
pupil_var = 0.2 + level * 0.1 + np.random.normal(0, 0.05)
blink_rate = 15 - level * 3 + np.random.normal(0, 2)
gaze_std = 0.15 + level * 0.1 + np.random.normal(0, 0.03)

features = np.array([mean_nn, rmssd, lf_hf, w,
pupil, pupil_var, blink_rate, gaze_std])
X_list.append(features)
y_list.append(level)

X = np.array(X_list)
y = np.array(y_list)
groups = np.repeat(np.arange(3), n_per_class)

# 训练
print("=== ECG+眼动双模态认知负荷分类器 ===\n")
clf.train(X, y, groups=np.repeat(np.arange(10), 9))

# 预测示例
print("\n=== 预测示例 ===")
# 生成测试数据
test_ecg = np.random.normal(0, 0.1, 15000)
test_ecg[np.random.choice(15000, 50, replace=False)] = 1.0
test_eye = np.column_stack([
np.random.normal(0.8, 0.05, 1800), # eye_openness
np.random.normal(0, 0.15, 1800), # gaze_x
np.random.normal(0, 0.15, 1800), # gaze_y
np.random.normal(3.8, 0.2, 1800) # pupil (overload)
])

result = clf.predict(test_ecg, test_eye)
print(f"预测: {result['level_name']} (置信度: {result['confidence']:.3f})")
print(f"概率: {result['probabilities']}")

论文1详细结果对比

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
"""
论文1各模态组合性能对比

数据来源: Xu et al., IEEE TIM 2026
"""

results = {
"三模态(β+HRV+瞳孔)": {"10fold": 85.76, "LOSO": 78.57, "标准": "最优"},
"双模态(β+HRV)": {"10fold": 79.32, "LOSO": 71.23, "标准": "去掉瞳孔"},
"双模态(β+瞳孔)": {"10fold": 80.15, "LOSO": 73.89, "标准": "去掉HRV"},
"双模态(HRV+瞳孔)": {"10fold": 73.28, "LOSO": 65.14, "标准": "去掉fNIRS(IMS场景)"},
"仅β(fNIRS)": {"10fold": 68.45, "LOSO": 58.92, "标准": "仅fNIRS"},
"仅HRV(ECG)": {"10fold": 61.23, "LOSO": 52.67, "标准": "仅ECG"},
"仅瞳孔(眼动)": {"10fold": 64.87, "LOSO": 56.34, "标准": "仅眼动"},
}

print(f"{'模态组合':<25} {'10-fold CV':<15} {'LOSO CV':<15} {'备注'}")
print("-" * 70)
for name, r in results.items():
print(f"{name:<25} {r['10fold']:.2f}% {r['LOSO']:.2f}% {r['标准']}")

print("\n=== IMS关键发现 ===")
print(f"去掉fNIRS后(HRV+瞳孔): 10-fold降至{results['双模态(HRV+瞳孔)']['10fold']}%")
print(f"比三模态下降: {85.76 - results['双模态(HRV+瞳孔)']['10fold']:.2f}个百分点")
print(f"LOSO下降: {78.57 - results['双模态(HRV+瞳孔)']['LOSO']:.2f}个百分点")
print(f"但仍是可用的基线(>65% 10-fold)")

IMS 落地建议

优先级 行动 预估准确率 投入
P0 ECG+眼动双模态基线 ~65-73% 3人月
P1 增加行为特征(方向盘+车道) ~75-80% 2人月
P2 个性化模型(5分钟自适应) +5-8% 3人月
P3 探索fNIRS替代(驾驶员帽子/眼镜) +5-10% 6人月

核心结论

  1. fNIRS是最重要但不可车载的模态 — β振幅CH18/CH20(左PFC BA10)是第一特征
  2. ECG+眼动可达65-73%准确率 — 可作为IMS认知负荷检测基线
  3. 瞳孔直径是最简单的单模态基线 — 75%准确率(论文2), 仅需DMS摄像头
  4. PFC激活模式反直觉 — 适中负荷时激活最低, 过载和低负荷都高
  5. SHAP可解释性对车规认证重要 — 需解释AI决策依据

写作日期: 2026-08-27
标签: fNIRS, ECG, 眼动, 多模态, 认知负荷, SHAP, 飞行员, IMS
论文来源: IEEE TIM 2026 + ICIEA 2026
IMS启示: 去掉fNIRS后ECG+眼动可达65-73%, 瞳孔单模态75%是简单基线