1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
| """
VLM 认证工作流核心逻辑复现
步骤：
1. 定义安全相关本体（ontology）
2. 使用结构化标注系统
3. 同义词评估处理语言变化
4. 统计量化幻觉率
5. 模型对比与部署决策
"""
from dataclasses import dataclass
from typing import List, Dict, Set
from enum import Enum
class SafetyCategory(Enum):
"""安全相关本体类别"""
PEDESTRIAN = "行人"
VEHICLE = "车辆"
CYCLIST = "骑行者"
TRAFFIC_SIGN = "交通标志"
TRAFFIC_LIGHT = "交通灯"
ROAD_MARKING = "道路标线"
OBSTACLE = "障碍物"
@dataclass
class OntologyNode:
"""本体节点"""
category: SafetyCategory
canonical_name: str
synonyms: List[str]
safety_critical: bool
@dataclass
class VLMResponse:
"""VLM 模型响应"""
model_name: str
image_id: str
detected_objects: List[str]
ground_truth: List[str]
confidence: float
class VLMQualifier:
"""VLM 设计认证工作流"""
def __init__(self, ontology: List[OntologyNode]):
self.ontology = ontology
self.synonym_map = self._build_synonym_map()
def _build_synonym_map(self) -> Dict[str, str]:
"""构建同义词映射"""
mapping = {}
for node in self.ontology:
mapping[node.canonical_name.lower()] = \
node.canonical_name
for syn in node.synonyms:
mapping[syn.lower()] = node.canonical_name
return mapping
def normalize_response(self,
response: List[str]) -> Set[str]:
"""
将 VLM 响应规范化为本体标准名
示例：
- "a person walking" → "Pedestrian"
- "a car" → "Vehicle"
- "someone on a bike" → "Cyclist"
"""
normalized = set()
for obj in response:
obj_lower = obj.lower()
matched = False
for syn, canonical in self.synonym_map.items():
if syn in obj_lower:
normalized.add(canonical)
matched = True
break
if not matched:
normalized.add(f"UNKNOWN:{obj}")
return normalized
def evaluate_hallucination(
self, response: VLMResponse
) -> Dict:
"""
评估 VLM 响应的幻觉
假阳性: VLM 报告但真值中没有
假阴性: 真值中有但 VLM 未报告
"""
predicted = self.normalize_response(
response.detected_objects
)
truth = self.normalize_response(
response.ground_truth
)
false_positives = predicted - truth
false_negatives = truth - predicted
true_positives = predicted & truth
precision = len(true_positives) / max(
len(predicted), 1
)
recall = len(true_positives) / max(
len(truth), 1
)
f1 = 2 * precision * recall / max(
precision + recall, 1e-10
)
safety_critical_misses = [
obj for obj in false_negatives
if self._is_safety_critical(obj)
]
return {
'model': response.model_name,
'image': response.image_id,
'true_positives': true_positives,
'false_positives': false_positives,
'false_negatives': false_negatives,
'precision': precision,
'recall': recall,
'f1': f1,
'safety_critical_misses': safety_critical_misses
}
def _is_safety_critical(self, obj_name: str) -> bool:
"""判断是否安全关键物体"""
for node in self.ontology:
if (node.canonical_name == obj_name and
node.safety_critical):
return True
return False
def aggregate_results(
self, results: List[Dict]
) -> Dict:
"""聚合多个评估结果"""
n = len(results)
avg_precision = sum(
r['precision'] for r in results
) / max(n, 1)
avg_recall = sum(
r['recall'] for r in results
) / max(n, 1)
avg_f1 = sum(
r['f1'] for r in results
) / max(n, 1)
total_fp = sum(
len(r['false_positives']) for r in results
)
total_fn = sum(
len(r['false_negatives']) for r in results
)
total_safety_misses = sum(
len(r['safety_critical_misses']) for r in results
)
return {
'num_evaluations': n,
'avg_precision': avg_precision,
'avg_recall': avg_recall,
'avg_f1': avg_f1,
'total_false_positives': total_fp,
'total_false_negatives': total_fn,
'total_safety_critical_misses': total_safety_misses,
'hallucination_rate': (total_fp + total_fn) / max(
n, 1
)
}
if __name__ == "__main__":
ontology = [
OntologyNode(
SafetyCategory.PEDESTRIAN, "Pedestrian",
["person", "walker", "pedestrian"],
safety_critical=True
),
OntologyNode(
SafetyCategory.VEHICLE, "Vehicle",
["car", "truck", "bus", "vehicle"],
safety_critical=True
),
OntologyNode(
SafetyCategory.CYCLIST, "Cyclist",
["bicycle", "bike", "cyclist"],
safety_critical=True
),
OntologyNode(
SafetyCategory.TRAFFIC_LIGHT, "TrafficLight",
["traffic light", "traffic signal"],
safety_critical=True
),
]
qualifier = VLMQualifier(ontology)
responses = [
VLMResponse(
model_name="VLM-A",
image_id="img_001",
detected_objects=[
"a person walking", "a car", "a traffic light"
],
ground_truth=[
"person", "vehicle", "traffic light"
],
confidence=0.85
),
VLMResponse(
model_name="VLM-A",
image_id="img_002",
detected_objects=[
"a car", "a truck", "someone on a bike"
],
ground_truth=[
"vehicle", "cyclist"
],
confidence=0.90
),
VLMResponse(
model_name="VLM-A",
image_id="img_003",
detected_objects=[
"a person", "a car"
],
ground_truth=[
"person", "vehicle", "cyclist"
],
confidence=0.75
),
]
results = []
for resp in responses:
result = qualifier.evaluate_hallucination(resp)
results.append(result)
print(f"\n=== {resp.model_name} @ {resp.image_id} ===")
print(f" 预测: {result['true_positives']}")
print(f" 假阳性: {result['false_positives']}")
print(f" 假阴性: {result['false_negatives']}")
print(f" 精确率: {result['precision']:.2f}")
print(f" 召回率: {result['recall']:.2f}")
print(f" F1: {result['f1']:.2f}")
if result['safety_critical_misses']:
print(f" ⚠️ 安全关键遗漏: "
f"{result['safety_critical_misses']}")
summary = qualifier.aggregate_results(results)
print(f"\n=== 聚合结果 ({summary['num_evaluations']} 次) ===")
print(f" 平均精确率: {summary['avg_precision']:.2f}")
print(f" 平均召回率: {summary['avg_recall']:.2f}")
print(f" 平均 F1: {summary['avg_f1']:.2f}")
print(f" 总假阳性: {summary['total_false_positives']}")
print(f" 总假阴性: {summary['total_false_negatives']}")
print(f" 幻觉率: {summary['hallucination_rate']:.2f}")
print(f" 安全关键遗漏: "
f"{summary['total_safety_critical_misses']}")
|
