1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
| """
VLM 汽车感知系统化认证框架
核心步骤：
1. 定义汽车感知任务测试集
2. 用确定性方法量化 VLM 幻觉
3. 比较不同 VLM 模型
4. 支持部署决策
"""
from dataclasses import dataclass
from typing import List, Tuple
import numpy as np
@dataclass
class PerceptionTask:
"""汽车感知任务定义"""
task_id: str
image: str
prompt: str
ground_truth: List[str]
category: str
@dataclass
class VLMResponse:
"""VLM 响应"""
detected_objects: List[str]
confidence: List[float]
response_text: str
class VLMQualifier:
"""
VLM 幻觉量化器
指标：
1. Hallucination Rate: VLM 报告不存在的物体比例
2. Miss Rate: VLM 漏检存在物体的比例
3. Precision: 正确报告比例
4. Recall: 检测到的真实物体比例
"""
def evaluate(self, task: PerceptionTask,
response: VLMResponse) -> dict:
"""评估单次响应"""
gt_set = set(task.ground_truth)
resp_set = set(response.detected_objects)
hallucinations = resp_set - gt_set
misses = gt_set - resp_set
correct = resp_set & gt_set
precision = len(correct) / len(resp_set) if resp_set else 0
recall = len(correct) / len(gt_set) if gt_set else 0
return {
'task_id': task.task_id,
'hallucinations': list(hallucinations),
'misses': list(misses),
'correct': list(correct),
'precision': precision,
'recall': recall,
'hallucination_rate': len(hallucinations) / max(
len(resp_set), 1),
}
def batch_evaluate(self, tasks: List[PerceptionTask],
responses: List[VLMResponse]) -> dict:
"""批量评估"""
results = []
for task, resp in zip(tasks, responses):
results.append(self.evaluate(task, resp))
avg_precision = np.mean([r['precision'] for r in results])
avg_recall = np.mean([r['recall'] for r in results])
avg_hallucination = np.mean([
r['hallucination_rate'] for r in results
])
by_category = {}
for r in results:
cat = next(
t.category for t in tasks
if t.task_id == r['task_id']
)
if cat not in by_category:
by_category[cat] = []
by_category[cat].append(r)
category_stats = {}
for cat, cat_results in by_category.items():
category_stats[cat] = {
'precision': np.mean([
r['precision'] for r in cat_results
]),
'recall': np.mean([
r['recall'] for r in cat_results
]),
'hallucination_rate': np.mean([
r['hallucination_rate'] for r in cat_results
]),
'count': len(cat_results),
}
return {
'total_tasks': len(tasks),
'avg_precision': avg_precision,
'avg_recall': avg_recall,
'avg_hallucination_rate': avg_hallucination,
'by_category': category_stats,
}
if __name__ == "__main__":
qualifier = VLMQualifier()
tasks = [
PerceptionTask(
task_id="cabin_001",
image="cabin_interior_001.jpg",
prompt="List all objects and people visible in this vehicle cabin.",
ground_truth=["driver", "front_passenger", "steering_wheel",
"dashboard", "rearview_mirror"],
category="cabin_occupant"
),
PerceptionTask(
task_id="cabin_002",
image="cabin_rear_002.jpg",
prompt="Is there a child in the rear seat?",
ground_truth=["child_seat", "child"],
category="cpd"
),
PerceptionTask(
task_id="dms_001",
image="dms_driver_001.jpg",
prompt="Describe the driver's state: eyes open/closed,
gaze direction, phone use.",
ground_truth=["eyes_open", "looking_road", "no_phone"],
category="driver_state"
),
]
responses = [
VLMResponse(
detected_objects=["driver", "front_passenger",
"steering_wheel", "phone_on_dashboard"],
confidence=[0.95, 0.90, 0.99, 0.60],
response_text="I see a driver, front passenger,
steering wheel, and phone on dashboard."
),
VLMResponse(
detected_objects=["child", "child_seat", "toy"],
confidence=[0.88, 0.95, 0.45],
response_text="There is a child in a child seat with a toy."
),
VLMResponse(
detected_objects=["eyes_open", "looking_road",
"holding_phone"],
confidence=[0.92, 0.85, 0.55],
response_text="Driver has eyes open, looking at road,
appears to be holding a phone."
),
]
results = qualifier.batch_evaluate(tasks, responses)
print("=== VLM 幻觉量化结果 ===")
print(f" 总任务数: {results['total_tasks']}")
print(f" 平均精确率: {results['avg_precision']:.2%}")
print(f" 平均召回率: {results['avg_recall']:.2%}")
print(f" 平均幻觉率: {results['avg_hallucination_rate']:.2%}")
print(f"\n=== 分类别统计 ===")
for cat, stats in results['by_category'].items():
print(f" {cat}:")
print(f" 精确率: {stats['precision']:.2%}")
print(f" 召回率: {stats['recall']:.2%}")
print(f" 幻觉率: {stats['hallucination_rate']:.2%}")
print(f" 样本数: {stats['count']}")
|
