视觉语言模型(VLM)用于车内场景理解：ISU-Test测试框架

论文信息

• 标题： Search-based Testing of Vision Language Models for In-Car Scene Understanding
• 发表： arXiv 2025
• 链接： https://arxiv.org/abs/2607.02300

研究背景

传统 DMS/OMS 系统基于特定任务训练的 CNN 模型，存在以下问题：

• 泛化能力差，难以处理未知场景
• 需要大量标注数据
• 缺乏语义理解能力

视觉语言模型（VLM）如 GPT-4V、Qwen-VL 等提供了通用视觉理解能力，可解读车内场景并输出结构化信息。

ISU-Test 框架

核心思想

ISU-Test 通过搜索式测试系统性地探索车内场景空间，发现 VLM 的失败案例。

class ISUTest:
    """ISU-Test: 车内场景理解测试框架"""
    
    def __init__(self, vlm_model, renderer):
        self.vlm = vlm_model
        self.renderer = renderer
        self.test_cases = []
    
    def generate_scene(self, params: dict) -> np.ndarray:
        """
        生成车内场景图像
        
        Args:
            params: {
                'driver_pose': str,  # normal/lean_forward/recline
                'objects': list,     # 物品列表
                'lighting': float,   # 光照强度
                'seatbelt': bool,    # 安全带状态
            }
        
        Returns:
            image: 生成的场景图像
        """
        return self.renderer.render(params)
    
    def query_vlm(self, image, question: str) -> str:
        """
        查询 VLM 模型
        
        Args:
            image: 输入图像
            question: 问题文本
        
        Returns:
            answer: VLM 回答
        """
        return self.vlm.generate(image, question)
    
    def search_failure_cases(self, 
                             param_bounds: dict,
                             target_behavior: str,
                             max_iterations: int = 1000):
        """
        搜索失败案例
        
        Args:
            param_bounds: 参数边界
            target_behavior: 目标行为（如"检测分心"）
            max_iterations: 最大迭代次数
        """
        failures = []
        
        for i in range(max_iterations):
            # 采样参数
            params = self._sample_params(param_bounds)
            
            # 生成场景
            image = self.generate_scene(params)
            
            # 查询 VLM
            question = self._construct_question(target_behavior)
            answer = self.query_vlm(image, question)
            
            # 验证答案
            ground_truth = self._get_ground_truth(params)
            if not self._is_correct(answer, ground_truth):
                failures.append({
                    'params': params,
                    'image': image,
                    'answer': answer,
                    'ground_truth': ground_truth
                })
        
        return failures
    
    def _sample_params(self, bounds):
        """采样参数空间"""
        import random
        params = {}
        for key, (min_val, max_val) in bounds.items():
            if isinstance(min_val, float):
                params[key] = random.uniform(min_val, max_val)
            elif isinstance(min_val, int):
                params[key] = random.randint(min_val, max_val)
            elif isinstance(min_val, bool):
                params[key] = random.choice([True, False])
        return params
    
    def _construct_question(self, behavior: str) -> str:
        """构建问题文本"""
        questions = {
            'distraction': "Is the driver distracted? If yes, describe the distraction type.",
            'seatbelt': "Is the driver wearing a seatbelt correctly?",
            'posture': "Describe the driver's posture and identify if it's abnormal.",
            'objects': "List all objects in the cabin and their positions."
        }
        return questions.get(behavior, "Describe the in-cabin scene.")

场景生成器

import numpy as np
from typing import List, Dict

class InCabinSceneGenerator:
    """车内场景生成器"""
    
    def __init__(self, config: dict):
        self.config = config
        # 3D 渲染引擎（如 NVIDIA Omniverse）
        self.renderer = self._init_renderer()
        
        # 预定义资产
        self.assets = {
            'drivers': ['male_01', 'female_01', 'elderly_01'],
            'poses': ['normal', 'lean_forward', 'recline', 'turn_left'],
            'objects': ['phone', 'bottle', 'bag', 'luggage'],
            'vehicles': ['sedan', 'suv', 'truck']
        }
    
    def render(self, params: Dict) -> np.ndarray:
        """
        渲染车内场景
        
        Args:
            params: 场景参数
        
        Returns:
            image: RGB 图像 (H, W, 3)
        """
        # 设置驾驶姿态
        self._set_driver_pose(params.get('driver_pose', 'normal'))
        
        # 放置物品
        self._place_objects(params.get('objects', []))
        
        # 设置光照
        self._set_lighting(params.get('lighting', 500))
        
        # 设置安全带
        self._set_seatbelt(params.get('seatbelt', True))
        
        # 渲染
        image = self.renderer.render()
        
        return image
    
    def _set_driver_pose(self, pose: str):
        """设置驾驶员姿态"""
        pose_angles = {
            'normal': {'torso': 0, 'head': 0, 'arms': 0},
            'lean_forward': {'torso': 25, 'head': 15, 'arms': 0},
            'recline': {'torso': -20, 'head': -10, 'arms': 0},
            'turn_left': {'torso': 0, 'head': 30, 'arms': 10}
        }
        
        angles = pose_angles.get(pose, pose_angles['normal'])
        self.renderer.set_skeleton_angles(angles)
    
    def _place_objects(self, objects: List[Dict]):
        """放置物品"""
        for obj in objects:
            obj_type = obj.get('type')
            position = obj.get('position', [0, 0, 0])
            rotation = obj.get('rotation', [0, 0, 0])
            
            self.renderer.place_object(obj_type, position, rotation)
    
    def _set_lighting(self, intensity: float):
        """设置光照"""
        self.renderer.set_light_intensity(intensity)
    
    def _set_seatbelt(self, worn: bool):
        """设置安全带"""
        self.renderer.set_seatbelt_state(worn)


class VLMInterface:
    """VLM 模型接口"""
    
    def __init__(self, model_name: str = 'gpt-4-vision'):
        self.model_name = model_name
        self.api_key = self._load_api_key()
    
    def generate(self, image: np.ndarray, question: str) -> str:
        """
        生成回答
        
        Args:
            image: 输入图像
            question: 问题文本
        
        Returns:
            answer: 生成的回答
        """
        # 编码图像
        image_base64 = self._encode_image(image)
        
        # 构建提示词
        prompt = self._build_prompt(question)
        
        # 调用 API
        response = self._call_api(image_base64, prompt)
        
        return response
    
    def _build_prompt(self, question: str) -> str:
        """构建提示词"""
        return f"""Analyze the in-car scene and answer the question.

Context: This is a driver monitoring system for automotive safety.

Question: {question}

Please provide a structured answer with:
1. Direct answer (Yes/No/Description)
2. Confidence level (High/Medium/Low)
3. Supporting observations"""
    
    def parse_answer(self, response: str) -> dict:
        """解析回答"""
        # 提取结构化信息
        lines = response.strip().split('\n')
        
        result = {
            'answer': '',
            'confidence': 'Medium',
            'observations': []
        }
        
        for line in lines:
            if line.startswith('Answer:'):
                result['answer'] = line.replace('Answer:', '').strip()
            elif line.startswith('Confidence:'):
                result['confidence'] = line.replace('Confidence:', '').strip()
            elif line.startswith('-'):
                result['observations'].append(line[1:].strip())
        
        return result

实验结果

失败案例分析

场景类型	失败率	主要失败原因
低光照	18.5%	视觉特征提取失败
遮挡	15.2%	无法推理隐藏部分
复杂物品组合	12.8%	注意力分散
边缘姿态	10.3%	训练数据不足

改进建议

class VLMRobustnessImprover:
    """VLM 鲁棒性改进"""
    
    @staticmethod
    def add_prompt_engineering(question: str) -> str:
        """提示词工程增强"""
        enhanced_prompt = f"""
{question}

Analysis guidelines:
1. If lighting is poor, state uncertainty explicitly
2. If objects are partially occluded, describe visible parts
3. Consider common driving scenarios and edge cases
4. When uncertain, provide confidence estimation

Output format:
- Status: [Clear/Warning/Critical]
- Confidence: [High/Medium/Low]
- Details: [Specific observations]
"""
        return enhanced_prompt
    
    @staticmethod
    def multi_turn_reasoning(vlm, image, initial_question):
        """多轮推理"""
        # 第一轮：整体描述
        scene_desc = vlm.generate(image, "Describe the in-cabin scene.")
        
        # 第二轮：具体问题
        answer = vlm.generate(image, initial_question)
        
        # 第三轮：验证
        verification = vlm.generate(image, 
            f"You answered '{answer}'. Is this consistent with the scene: {scene_desc}?")
        
        return {
            'scene_description': scene_desc,
            'initial_answer': answer,
            'verification': verification
        }

IMS 开发启示

1. VLM 用于 DMS/OMS 的优势

传统 CNN	VLM
单一任务	通用理解
固定输出	自然语言交互
大量标注	零样本学习
黑盒决策	可解释输出

2. 部署建议

# VLM 车载部署方案
VLM_DEPLOYMENT_OPTIONS = {
    'cloud_based': {
        'model': 'GPT-4V / Claude 3.5',
        'latency': '500-2000ms',
        'privacy': '低',
        'cost': '高'
    },
    'edge_optimized': {
        'model': 'Qwen-VL-Chat (量化)',
        'latency': '100-300ms',
        'privacy': '高',
        'cost': '中'
    },
    'hybrid': {
        'model': '本地 Qwen-VL + 云端大模型',
        'latency': '50-200ms (本地)',
        'privacy': '中',
        'cost': '中'
    }
}

3. 测试场景定义

# Euro NCAP 2026 VLM 测试场景
VLM_TEST_SCENARIOS = [
    {
        'id': 'D-01',
        'name': '手机使用检测',
        'question': 'Is the driver using a phone?',
        'pass_criteria': '检测准确率 ≥95%'
    },
    {
        'id': 'D-02',
        'name': '分心检测',
        'question': 'Is the driver looking away from the road?',
        'pass_criteria': '检测延迟 ≤3s'
    },
    {
        'id': 'S-01',
        'name': '安全带检测',
        'question': 'Is the seatbelt correctly worn?',
        'pass_criteria': '误报率 ≤5%'
    },
    {
        'id': 'O-01',
        'name': '乘员检测',
        'question': 'How many occupants are in the cabin?',
        'pass_criteria': '计数准确率 ≥98%'
    }
]

总结

ISU-Test 框架通过搜索式测试系统性发现 VLM 在车内场景理解中的失败案例，为 VLM 在 DMS/OMS 中的应用提供了验证方法。结合提示词工程和多轮推理，可提升 VLM 鲁棒性，满足车载安全要求。

---

参考链接：

• 论文原文：https://arxiv.org/abs/2607.02300
• Qwen-VL：https://github.com/QwenLM/Qwen-VL
• LLM4AD 资源：https://github.com/Thinklab-SJTU/Awesome-LLM4AD