comprehensive_frontend_test.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
前端MCP服务综合测试脚本 V2.1.1
专注于React和Vue生态系统，使用优化的评分算法测试MCP服务
评分维度：清晰性、约束性、相关性
"""

import time
import json
from typing import List, Dict, Any
from src.engine import RuleMatchEngine

class ReactVueTestSuite:
    def __init__(self):
        self.engine = RuleMatchEngine()
        self.test_cases = self._generate_test_cases()
        self.results = []

    def _generate_test_cases(self) -> List[Dict[str, Any]]:
        """生成30个React和Vue生态测试用例"""
        return [
            # React基础组件开发 (1-5)
            {
                "id": 1,
                "query": "帮我用React和TypeScript创建一个用户登录表单组件",
                "expected_tech": ["react", "typescript", "component"],
                "tech_stack": "React",
                "category": "React组件开发",
                "complexity": "medium"
            },
            {
                "id": 2,
                "query": "React项目中添加刷新按钮，使用Ant Design组件库",
                "expected_tech": ["react", "component", "antd"],
                "tech_stack": "React",
                "category": "React组件开发",
                "complexity": "low"
            },
            {
                "id": 3,
                "query": "React函数组件中创建可复用的模态框组件",
                "expected_tech": ["react", "component", "modal"],
                "tech_stack": "React",
                "category": "React组件开发",
                "complexity": "medium"
            },
            {
                "id": 4,
                "query": "React中使用forwardRef创建可引用的输入组件",
                "expected_tech": ["react", "forwardRef", "component"],
                "tech_stack": "React",
                "category": "React组件开发",
                "complexity": "medium"
            },
            {
                "id": 5,
                "query": "React项目中使用CSS Modules进行样式隔离",
                "expected_tech": ["react", "css", "modules"],
                "tech_stack": "React",
                "category": "React样式",
                "complexity": "medium"
            },

            # React Hooks和状态管理 (6-10)
            {
                "id": 6,
                "query": "React项目中如何使用useState和useEffect管理组件状态",
                "expected_tech": ["react", "hooks"],
                "tech_stack": "React",
                "category": "React状态管理",
                "complexity": "medium"
            },
            {
                "id": 7,
                "query": "React函数组件中使用useContext实现全局状态共享",
                "expected_tech": ["react", "hooks", "context"],
                "tech_stack": "React",
                "category": "React状态管理",
                "complexity": "high"
            },
            {
                "id": 8,
                "query": "React项目中使用Redux Toolkit进行状态管理",
                "expected_tech": ["react", "redux", "toolkit"],
                "tech_stack": "React",
                "category": "React状态管理",
                "complexity": "high"
            },
            {
                "id": 9,
                "query": "React中使用Zustand轻量级状态管理库",
                "expected_tech": ["react", "zustand", "state"],
                "tech_stack": "React",
                "category": "React状态管理",
                "complexity": "medium"
            },
            {
                "id": 10,
                "query": "React自定义Hook封装数据获取逻辑",
                "expected_tech": ["react", "hooks", "custom"],
                "tech_stack": "React",
                "category": "React状态管理",
                "complexity": "high"
            },

            # React路由和性能优化 (11-15)
            {
                "id": 11,
                "query": "在React应用中集成React Router进行页面路由配置",
                "expected_tech": ["react", "router"],
                "tech_stack": "React",
                "category": "React路由",
                "complexity": "medium"
            },
            {
                "id": 12,
                "query": "React应用性能优化：使用React.memo和useMemo减少重渲染",
                "expected_tech": ["react", "performance", "hooks"],
                "tech_stack": "React",
                "category": "React性能优化",
                "complexity": "high"
            },
            {
                "id": 13,
                "query": "React组件单元测试编写，使用Jest和React Testing Library",
                "expected_tech": ["react", "testing", "jest"],
                "tech_stack": "React",
                "category": "React测试",
                "complexity": "high"
            },
            {
                "id": 14,
                "query": "React项目中使用styled-components进行CSS-in-JS开发",
                "expected_tech": ["react", "styled", "css"],
                "tech_stack": "React",
                "category": "React样式",
                "complexity": "medium"
            },
            {
                "id": 15,
                "query": "React应用中实现代码分割和懒加载优化",
                "expected_tech": ["react", "lazy", "performance"],
                "tech_stack": "React",
                "category": "React性能优化",
                "complexity": "high"
            },

            # Vue基础组件开发 (16-20)
            {
                "id": 16,
                "query": "Vue 3项目中使用Composition API创建响应式数据表格组件",
                "expected_tech": ["vue", "composition", "component"],
                "tech_stack": "Vue",
                "category": "Vue组件开发",
                "complexity": "medium"
            },
            {
                "id": 17,
                "query": "Vue 3中使用Teleport实现模态框组件",
                "expected_tech": ["vue", "teleport", "component"],
                "tech_stack": "Vue",
                "category": "Vue组件开发",
                "complexity": "medium"
            },
            {
                "id": 18,
                "query": "Vue组件生命周期钩子的使用和最佳实践",
                "expected_tech": ["vue", "lifecycle"],
                "tech_stack": "Vue",
                "category": "Vue基础",
                "complexity": "medium"
            },
            {
                "id": 19,
                "query": "Vue 3中使用defineProps和defineEmits定义组件接口",
                "expected_tech": ["vue", "props", "emits"],
                "tech_stack": "Vue",
                "category": "Vue组件开发",
                "complexity": "medium"
            },
            {
                "id": 20,
                "query": "Vue项目中使用Scoped CSS实现样式隔离",
                "expected_tech": ["vue", "scoped", "css"],
                "tech_stack": "Vue",
                "category": "Vue样式",
                "complexity": "medium"
            },

            # Vue状态管理 (21-25)
            {
                "id": 21,
                "query": "Vue应用中使用Vuex进行状态管理和数据流控制",
                "expected_tech": ["vue", "vuex", "state"],
                "tech_stack": "Vue",
                "category": "Vue状态管理",
                "complexity": "high"
            },
            {
                "id": 22,
                "query": "Vue项目中使用Pinia替代Vuex进行状态管理",
                "expected_tech": ["vue", "pinia", "state"],
                "tech_stack": "Vue",
                "category": "Vue状态管理",
                "complexity": "medium"
            },
            {
                "id": 23,
                "query": "Vue 3中使用provide/inject实现跨组件通信",
                "expected_tech": ["vue", "provide", "inject"],
                "tech_stack": "Vue",
                "category": "Vue状态管理",
                "complexity": "medium"
            },
            {
                "id": 24,
                "query": "Vue Composition API中使用reactive和ref管理响应式数据",
                "expected_tech": ["vue", "reactive", "ref"],
                "tech_stack": "Vue",
                "category": "Vue状态管理",
                "complexity": "medium"
            },
            {
                "id": 25,
                "query": "Vue自定义组合式函数封装业务逻辑",
                "expected_tech": ["vue", "composables", "custom"],
                "tech_stack": "Vue",
                "category": "Vue状态管理",
                "complexity": "high"
            },

            # Vue路由和测试 (26-30)
            {
                "id": 26,
                "query": "Vue Router配置动态路由和路由守卫",
                "expected_tech": ["vue", "router"],
                "tech_stack": "Vue",
                "category": "Vue路由",
                "complexity": "medium"
            },
            {
                "id": 27,
                "query": "Vue组件单元测试编写，使用Vue Test Utils",
                "expected_tech": ["vue", "testing", "utils"],
                "tech_stack": "Vue",
                "category": "Vue测试",
                "complexity": "high"
            },
            {
                "id": 28,
                "query": "Vue 3项目性能优化：使用v-memo和异步组件",
                "expected_tech": ["vue", "performance", "memo"],
                "tech_stack": "Vue",
                "category": "Vue性能优化",
                "complexity": "high"
            },
            {
                "id": 29,
                "query": "Vue项目中使用TypeScript进行类型安全开发",
                "expected_tech": ["vue", "typescript"],
                "tech_stack": "Vue",
                "category": "Vue类型安全",
                "complexity": "medium"
            },
            {
                "id": 30,
                "query": "Vue 3中使用Suspense处理异步组件加载",
                "expected_tech": ["vue", "suspense", "async"],
                "tech_stack": "Vue",
                "category": "Vue组件开发",
                "complexity": "high"
            }
        ]

    def _evaluate_accuracy(self, result: str, expected_tech: List[str], tech_stack: str = "") -> float:
        """基于清晰性、约束性、相关性三个维度评估规范质量"""
        if not result or "未找到" in result:
            return 0.0

        # 清晰性评分 (1-5分)
        clarity_score = self._evaluate_clarity(result)

        # 约束性评分 (1-5分)
        constraint_score = self._evaluate_constraint(result)

        # 相关性评分 (1-5分)
        relevance_score = self._evaluate_relevance(result, expected_tech, tech_stack)

        # 综合评分
        total_score = (clarity_score + constraint_score + relevance_score) / 3

        # 返回0-1范围的分数
        return min(total_score / 5.0, 1.0)

    def _evaluate_clarity(self, content: str) -> float:
        """评估规范的清晰性 (1-5分)"""
        if not content:
            return 1.0

        score = 1.0

        # 长度适中 (+1分)
        if 20 <= len(content) <= 500:
            score += 1.0

        # 包含具体的技术术语 (+1分)
        tech_terms = ['组件', '函数', '方法', '属性', '接口', '类型', '状态', '路由', '测试', 'API']
        if any(term in content for term in tech_terms):
            score += 1.0

        # 避免模糊词汇 (+1分)
        vague_words = ['可能', '也许', '大概', '应该会', '或许']
        if not any(word in content for word in vague_words):
            score += 1.0

        # 结构清晰，有明确的指导 (+1分)
        clear_indicators = ['必须', '应该', '禁止', '推荐', '避免', '使用', '不要']
        if any(indicator in content for indicator in clear_indicators):
            score += 1.0

        return min(score, 5.0)

    def _evaluate_constraint(self, content: str) -> float:
        """评估规范的约束性 (1-5分)"""
        if not content:
            return 1.0

        score = 1.0

        # 包含明确的行为指导 (+2分)
        action_words = ['必须', '禁止', '不得', '应该', '需要', '要求']
        if any(word in content for word in action_words):
            score += 2.0

        # 提供具体的实现方式 (+1分)
        implementation_words = ['使用', '通过', '采用', '实现', '配置', '设置']
        if any(word in content for word in implementation_words):
            score += 1.0

        # 有明确的标准或规则 (+1分)
        standard_words = ['规范', '标准', '约定', '规则', '原则', '最佳实践']
        if any(word in content for word in standard_words):
            score += 1.0

        return min(score, 5.0)

    def _evaluate_relevance(self, content: str, expected_tech: List[str], tech_stack: str) -> float:
        """评估规范的相关性 (1-5分)"""
        if not content:
            return 1.0

        score = 1.0
        content_lower = content.lower()

        # 技术栈匹配 (+2分)
        if tech_stack.lower() in content_lower:
            score += 2.0

        # 预期技术关键词匹配 (+2分)
        matched_tech = sum(1 for tech in expected_tech if tech.lower() in content_lower)
        if matched_tech > 0:
            score += min(matched_tech * 0.5, 2.0)

        # 避免无关内容 (+1分)
        irrelevant_words = ['后端', 'backend', '数据库', '服务器', 'server']
        if not any(word in content_lower for word in irrelevant_words):
            score += 1.0

        return min(score, 5.0)

    def _evaluate_usefulness(self, result: str, category: str, complexity: str, tech_stack: str = "") -> float:
        """评估实用性评分（基于规范数量和整体质量）"""
        if not result or "未找到" in result:
            return 0.0

        # 基于内容长度的基础分数
        length_score = min(len(result) / 200.0, 1.0) if result else 0.0

        # 基于技术栈相关性的质量分数
        tech_relevance = 1.0 if tech_stack.lower() in result.lower() else 0.5

        # 基于复杂度的权重调整
        complexity_weight = {
            'simple': 0.8,
            'medium': 1.0,
            'high': 1.2,
            'very_high': 1.3
        }.get(complexity, 1.0)

        # 内容质量指标
        quality_indicators = ['最佳实践', '推荐', '避免', '注意', '建议', '规范', '标准']
        quality_score = sum(1 for indicator in quality_indicators if indicator in result) / len(quality_indicators)

        # 综合实用性评分
        usefulness = (length_score * 0.3 + tech_relevance * 0.4 + quality_score * 0.3) * complexity_weight
        return min(usefulness, 1.0)

    def _count_standards(self, result: str) -> int:
        """统计返回的规范数量"""
        if not result:
            return 0

        # 简单统计方法：按段落分割
        sections = result.split('\n\n')
        return len([s for s in sections if s.strip()])

    def run_single_test(self, test_case: Dict[str, Any]) -> Dict[str, Any]:
        """运行单个测试用例"""
        print(f"\n测试用例 {test_case['id']}: {test_case['query'][:50]}...")

        # 记录开始时间
        start_time = time.time()

        try:
            # 调用MCP服务
            result = self.engine.get_standards(test_case['query'])

            # 记录结束时间
            end_time = time.time()
            elapsed_ms = round((end_time - start_time) * 1000, 2)

            # 评估结果
            accuracy = self._evaluate_accuracy(result, test_case['expected_tech'], test_case['tech_stack'])
            usefulness = self._evaluate_usefulness(result, test_case['category'], test_case['complexity'], test_case['tech_stack'])
            standards_count = self._count_standards(result)

            test_result = {
                'id': test_case['id'],
                'query': test_case['query'],
                'category': test_case['category'],
                'complexity': test_case['complexity'],
                'expected_tech': test_case['expected_tech'],
                'elapsed_ms': elapsed_ms,
                'accuracy': round(accuracy, 2),
                'usefulness': usefulness,
                'standards_count': standards_count,
                'result_length': len(result) if result else 0,
                'success': bool(result and "未找到" not in result),
                'result_preview': result[:200] + '...' if result and len(result) > 200 else result
            }

            print(f"  ✓ 耗时: {elapsed_ms}ms | 准确率: {accuracy:.2f} | 实用性: {usefulness}/5 | 规范数: {standards_count}")

            return test_result

        except Exception as e:
            end_time = time.time()
            elapsed_ms = round((end_time - start_time) * 1000, 2)

            error_result = {
                'id': test_case['id'],
                'query': test_case['query'],
                'category': test_case['category'],
                'complexity': test_case['complexity'],
                'expected_tech': test_case['expected_tech'],
                'elapsed_ms': elapsed_ms,
                'accuracy': 0.0,
                'usefulness': 1,
                'standards_count': 0,
                'result_length': 0,
                'success': False,
                'error': str(e),
                'result_preview': f"错误: {str(e)}"
            }

            print(f"  ✗ 错误: {str(e)}")
            return error_result

    def run_all_tests(self) -> List[Dict[str, Any]]:
        """运行所有测试用例"""
        print("=" * 80)
        print("前端MCP服务综合测试开始")
        print(f"总计测试用例: {len(self.test_cases)}")
        print("=" * 80)

        results = []

        for test_case in self.test_cases:
            result = self.run_single_test(test_case)
            results.append(result)

            # 短暂延迟，避免过快调用
            time.sleep(0.1)

        self.results = results
        return results

    def generate_report(self) -> str:
        """生成详细测试报告"""
        if not self.results:
            return "没有测试结果可生成报告"

        # 统计数据
        total_tests = len(self.results)
        successful_tests = len([r for r in self.results if r['success']])
        success_rate = (successful_tests / total_tests) * 100

        avg_elapsed = sum(r['elapsed_ms'] for r in self.results) / total_tests
        avg_accuracy = sum(r['accuracy'] for r in self.results) / total_tests
        avg_usefulness = sum(r['usefulness'] for r in self.results) / total_tests
        avg_standards_count = sum(r['standards_count'] for r in self.results) / total_tests

        # 按类别统计
        category_stats = {}
        for result in self.results:
            category = result['category']
            if category not in category_stats:
                category_stats[category] = {'count': 0, 'success': 0, 'avg_accuracy': 0, 'avg_elapsed': 0}

            category_stats[category]['count'] += 1
            if result['success']:
                category_stats[category]['success'] += 1
            category_stats[category]['avg_accuracy'] += result['accuracy']
            category_stats[category]['avg_elapsed'] += result['elapsed_ms']

        for category in category_stats:
            stats = category_stats[category]
            stats['success_rate'] = (stats['success'] / stats['count']) * 100
            stats['avg_accuracy'] = stats['avg_accuracy'] / stats['count']
            stats['avg_elapsed'] = stats['avg_elapsed'] / stats['count']

        # 生成报告
        report = f"""
=== 前端MCP服务测试报告 ===
测试时间: {time.strftime('%Y-%m-%d %H:%M:%S')}

📊 总体统计:
- 总测试用例: {total_tests}
- 成功测试: {successful_tests}
- 成功率: {success_rate:.1f}%
- 平均耗时: {avg_elapsed:.2f}ms
- 平均准确率: {avg_accuracy:.2f}
- 平均实用性: {avg_usefulness:.1f}
- 平均规范数量: {avg_standards_count:.1f}

📈 性能分析:
- 最快响应: {min(r['elapsed_ms'] for r in self.results):.2f}ms
- 最慢响应: {max(r['elapsed_ms'] for r in self.results):.2f}ms
- 响应时间中位数: {sorted([r['elapsed_ms'] for r in self.results])[total_tests//2]:.2f}ms

🎯 准确率分析:
- 最高准确率: {max(r['accuracy'] for r in self.results):.2f}
- 最低准确率: {min(r['accuracy'] for r in self.results):.2f}
- 完全匹配(准确率=1.0): {len([r for r in self.results if r['accuracy'] == 1.0])}/{total_tests}

📋 分类统计:
"""

        for category, stats in category_stats.items():
            report += f"""
{category}:
  - 测试数量: {stats['count']}
  - 成功率: {stats['success_rate']:.1f}%
  - 平均准确率: {stats['avg_accuracy']:.2f}
  - 平均耗时: {stats['avg_elapsed']:.2f}ms
"""

        # 失败用例分析
        failed_tests = [r for r in self.results if not r['success']]
        if failed_tests:
            report += f"\n❌ 失败用例分析 ({len(failed_tests)}个):\n"
            for test in failed_tests:
                report += f"  - 用例{test['id']}: {test['query'][:50]}...\n"
                report += f"    错误: {test.get('error', '未知错误')}\n"

        # 性能优异用例
        fast_tests = sorted(self.results, key=lambda x: x['elapsed_ms'])[:5]
        report += f"\n⚡ 响应最快的5个用例:\n"
        for i, test in enumerate(fast_tests, 1):
            report += f"  {i}. 用例{test['id']} - {test['elapsed_ms']:.2f}ms: {test['query'][:40]}...\n"

        # 准确率最高用例
        accurate_tests = sorted([r for r in self.results if r['success']],
                               key=lambda x: (x['accuracy'], -x['elapsed_ms']), reverse=True)[:5]
        report += f"\n🎯 准确率最高的5个用例:\n"
        for i, test in enumerate(accurate_tests, 1):
            report += f"  {i}. 用例{test['id']} - 准确率{test['accuracy']:.2f}: {test['query'][:40]}...\n"

        return report

    def save_detailed_results(self, filename: str = "frontend_test_results.json"):
        """保存详细测试结果到JSON文件"""
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump({
                'test_metadata': {
                    'test_time': time.strftime('%Y-%m-%d %H:%M:%S'),
                    'total_tests': len(self.results),
                    'mcp_version': 'V2.1.1'
                },
                'results': self.results
            }, f, ensure_ascii=False, indent=2)

        print(f"\n详细测试结果已保存到: {filename}")

def main():
    """主函数"""
    print("初始化前端MCP服务测试套件...")

    # 创建测试套件
    test_suite = ReactVueTestSuite()

    # 运行所有测试
    results = test_suite.run_all_tests()

    # 生成并显示报告
    report = test_suite.generate_report()
    print("\n" + "=" * 80)
    print(report)
    print("=" * 80)

    # 保存详细结果
    test_suite.save_detailed_results()

    print("\n测试完成！")
    print(f"成功率: {len([r for r in results if r['success']]) / len(results) * 100:.1f}%")
    print(f"平均耗时: {sum(r['elapsed_ms'] for r in results) / len(results):.2f}ms")
    print(f"平均准确率: {sum(r['accuracy'] for r in results) / len(results):.2f}")

if __name__ == "__main__":
  main()