MacBook Air M4 
内存: 16G 统一内存架构
推理: CPU + GPU (Metal) 混合

# 创建 conda 环境并安装依赖
conda create -n llmbench python=3.12
conda activate llmbench
pip install openai requests

# 安装 Ollama
brew install ollama        # macOS
# curl -fsSL https://ollama.com/install.sh | sh  # Linux
ollama serve

mkdir -p ~/Documents/GitHub/Others && cd ~/Documents/GitHub/Others

# BFCL — 工具调用评测事实标准
git clone https://github.com/ShishirPatil/gorilla.git bfcl
cd bfcl/berkeley-function-call-leaderboard
pip install -e .

# ToolBench — 大规模真实 API 工具评测
cd ~/Documents/GitHub/Others
git clone https://github.com/OpenBMB/ToolBench.git toolbench
cd toolbench
pip install -r requirements.txt

# τ-bench — Sierra 出品 Agent 业务场景评测
cd ~/Documents/GitHub/Others
git clone https://github.com/sierra-research/tau2-bench.git tau-bench
cd tau-bench
pip install uv && uv sync

# 回到工作目录
cd ~/Documents/GitHub/Others

# BFCL 配置 — 指向 Ollama 的 OpenAI 兼容端点
export OPENAI_API_KEY=ollama
export OPENAI_BASE_URL=http://localhost:11434/v1
export BFCL_PROJECT_ROOT=$HOME/Documents/GitHub/Others/bfcl/berkeley-function-call-leaderboard

# 批量拉取所有测试模型
ollama pull qwen3:0.6b
ollama pull qwen3.5:0.8b
ollama pull lfm2.5-thinking:latest
ollama pull granite4:350m-h
ollama pull granite4:350m
ollama pull functiongemma:latest
ollama pull llama3.2:1b
ollama pull smollm2:135m
ollama pull smollm2:360m
ollama pull smollm2:1.7b
ollama pull gemma3:1b

import requests, json, time, re

OLLAMA = "http://localhost:11434"

TOOLS = {
    "get_weather": {"type":"function","function":{"name":"get_weather",
        "description":"Get current weather / 查询城市天气",
        "parameters":{"type":"object",
            "properties":{"city":{"type":"string","description":"City name"},
                          "unit":{"type":"string","enum":["celsius","fahrenheit"]}},
            "required":["city"]}}},
    "get_time": {"type":"function","function":{"name":"get_time",
        "description":"Get current time for timezone / 获取时区时间",
        "parameters":{"type":"object",
            "properties":{"timezone":{"type":"string","description":"e.g. Asia/Shanghai"}},
            "required":["timezone"]}}},
    "search": {"type":"function","function":{"name":"search",
        "description":"Search information / 搜索信息",
        "parameters":{"type":"object",
            "properties":{"query":{"type":"string"},"limit":{"type":"integer"}},
            "required":["query"]}}},
    "send_email": {"type":"function","function":{"name":"send_email",
        "description":"Send email / 发送邮件",
        "parameters":{"type":"object",
            "properties":{"to":{"type":"string"},"subject":{"type":"string"},"body":{"type":"string"}},
            "required":["to","subject","body"]}}},
    "calculate": {"type":"function","function":{"name":"calculate",
        "description":"Math calculation / 数学计算",
        "parameters":{"type":"object",
            "properties":{"expression":{"type":"string","description":"e.g. '2+3*4'"}},
            "required":["expression"]}}},
    "translate": {"type":"function","function":{"name":"translate",
        "description":"Translate text / 翻译",
        "parameters":{"type":"object",
            "properties":{"text":{"type":"string"},"target_lang":{"type":"string"}},
            "required":["text","target_lang"]}}},
    "get_stock": {"type":"function","function":{"name":"get_stock",
        "description":"Get stock price / 获取股价",
        "parameters":{"type":"object",
            "properties":{"symbol":{"type":"string","description":"Ticker e.g. AAPL"}},
            "required":["symbol"]}}},
    "book_restaurant": {"type":"function","function":{"name":"book_restaurant",
        "description":"Book restaurant / 预订餐厅",
        "parameters":{"type":"object",
            "properties":{"restaurant":{"type":"string"},"date":{"type":"string"},
                          "time":{"type":"string"},"guests":{"type":"integer"}},
            "required":["restaurant","date","time","guests"]}}},
    "get_flight": {"type":"function","function":{"name":"get_flight",
        "description":"Search flights / 查询航班",
        "parameters":{"type":"object",
            "properties":{"origin":{"type":"string"},"destination":{"type":"string"},"date":{"type":"string"}},
            "required":["origin","destination","date"]}}},
    "get_order": {"type":"function","function":{"name":"get_order",
        "description":"Look up order / 查询订单",
        "parameters":{"type":"object",
            "properties":{"order_id":{"type":"string","description":"Order ID"}},
            "required":["order_id"]}}},
}

# 测试用例定义 — 23 个用例 × 2 语言 = 46 个测试/模型
TEST_CASES = [
    # === BFCL simple (6 cases) ===
    {"id":"s01","cat":"simple","desc":"weather by city",
     "en":"What's the weather in Beijing?","zh":"北京今天天气怎么样？",
     "tools":["get_weather"],"expect":[{"tool":"get_weather","args":{"city":"Beijing"}}]},
    {"id":"s02","cat":"simple","desc":"with optional param",
     "en":"Check weather in Tokyo, use Fahrenheit.","zh":"用华氏度查东京天气。",
     "tools":["get_weather"],"expect":[{"tool":"get_weather","args":{"city":"Tokyo","unit":"fahrenheit"}}]},
    {"id":"s03","cat":"simple","desc":"math calculation",
     "en":"Calculate (15 + 27) * 3.","zh":"帮我算一下 (15 + 27) * 3。",
     "tools":["calculate"],"expect":[{"tool":"calculate","args":{"expression":"(15 + 27) * 3"}}]},
    {"id":"s04","cat":"simple","desc":"stock lookup",
     "en":"What's the current price of AAPL?","zh":"查一下茅台股票的价格。",
     "tools":["get_stock"],"expect":[{"tool":"get_stock","args":{"symbol":"AAPL"}}]},
    {"id":"s05","cat":"simple","desc":"flight search",
     "en":"Find flights from SFO to JFK on 2026-09-01.","zh":"查北京到上海航班，9月1日。",
     "tools":["get_flight"],"expect":[{"tool":"get_flight","args":{"origin":"SFO","destination":"JFK","date":"2026-09-01"}}]},
    {"id":"s06","cat":"simple","desc":"order lookup",
     "en":"Look up order #ORD-12345.","zh":"查订单号 ORD-12345。",
     "tools":["get_order"],"expect":[{"tool":"get_order","args":{"order_id":"ORD-12345"}}]},

    # === BFCL parallel (4 cases) ===
    {"id":"p01","cat":"parallel","desc":"two cities weather",
     "en":"Weather in Paris and London?","zh":"同时查巴黎和伦敦天气。",
     "tools":["get_weather"],"expect":[{"tool":"get_weather","args":{"city":"Paris"}},{"tool":"get_weather","args":{"city":"London"}}]},
    {"id":"p02","cat":"parallel","desc":"weather + time",
     "en":"Tell me Tokyo weather and New York time.","zh":"告诉我东京天气和纽约时间。",
     "tools":["get_weather","get_time"],"expect":[{"tool":"get_weather","args":{"city":"Tokyo"}},{"tool":"get_time","args":{"timezone":"America/New_York"}}]},
    {"id":"p03","cat":"parallel","desc":"stock + search",
     "en":"Get NVDA stock price and search for NVIDIA news.","zh":"查NVDA股价同时搜NVIDIA新闻。",
     "tools":["get_stock","search"],"expect":[{"tool":"get_stock","args":{"symbol":"NVDA"}},{"tool":"search","args":{"query":"NVIDIA"}}]},
    {"id":"p04","cat":"parallel","desc":"three tools",
     "en":"Weather in Shanghai, time in Tokyo, search AI news.","zh":"同时查上海天气、东京时间、搜AI新闻。",
     "tools":["get_weather","get_time","search"],"expect":[{"tool":"get_weather","args":{"city":"Shanghai"}},{"tool":"get_time","args":{"timezone":"Asia/Tokyo"}},{"tool":"search","args":{"query":"AI"}}]},

    # === BFCL tool_selection (4 cases) ===
    {"id":"m01","cat":"tool_selection","desc":"time over weather/news",
     "en":"What time is it in London?","zh":"伦敦现在几点了？",
     "tools":["get_weather","get_time","search"],"expect":[{"tool":"get_time","args":{"timezone":"Europe/London"}}]},
    {"id":"m02","cat":"tool_selection","desc":"stock over weather/time",
     "en":"What's the current price of TSLA?","zh":"查一下特斯拉股价。",
     "tools":["get_weather","get_time","get_stock"],"expect":[{"tool":"get_stock","args":{"symbol":"TSLA"}}]},
    {"id":"m03","cat":"tool_selection","desc":"email (complex params)",
     "en":"Send email to fred@test.com, subject 'Meeting', body 'Room 301 at 3pm'.","zh":"给fred@test.com发邮件，主题会议，内容下午三点301。",
     "tools":["get_weather","send_email","calculate","search"],"expect":[{"tool":"send_email","args":{"to":"fred@test.com","subject":"Meeting","body":"Room 301 at 3pm"}}]},
    {"id":"m04","cat":"tool_selection","desc":"flight among 5 tools",
     "en":"Book a flight from SFO to JFK on 2026-08-20.","zh":"订8月20日北京到上海机票。",
     "tools":["get_weather","get_time","search","get_stock","get_flight"],"expect":[{"tool":"get_flight","args":{"origin":"SFO","destination":"JFK","date":"2026-08-20"}}]},

    # === BFCL irrelevant (2 cases) ===
    {"id":"i01","cat":"irrelevant","desc":"book among 8 tools",
     "en":"Book a table for 4 at Chez Panisse on 2026-08-15 at 7pm.","zh":"在鼎泰丰预订8月15日晚7点，4位。",
     "tools":["get_weather","get_time","search","send_email","calculate","translate","get_stock","book_restaurant"],
     "expect":[{"tool":"book_restaurant","args":{"restaurant":"Chez Panisse","date":"2026-08-15","time":"19:00","guests":4}}]},
    {"id":"i02","cat":"irrelevant","desc":"calc among 7 tools",
     "en":"What is 12345 * 67890?","zh":"12345乘以67890是多少？",
     "tools":["get_weather","get_time","search","send_email","translate","get_stock","book_restaurant","calculate"],
     "expect":[{"tool":"calculate","args":{"expression":"12345 * 67890"}}]},

    # === BFCL multi_turn (2 cases) ===
    {"id":"t01","cat":"multi_turn","desc":"translate then email","ordered":True,
     "en":"Translate 'Hello, how are you?' to Chinese, then email to test@example.com subject 'Translation'.",
     "zh":"把'Hello, how are you?'翻译成中文，发邮件给test@example.com，主题翻译结果。",
     "tools":["translate","send_email"],"expect":[{"tool":"translate","args":{"text":"Hello, how are you?","target_lang":"zh"}},{"tool":"send_email","args":{"to":"test@example.com","subject":"Translation"}}]},
    {"id":"t02","cat":"multi_turn","desc":"weather→translate→email","ordered":True,
     "en":"Get Shanghai weather, translate to Japanese, email to info@example.com.","zh":"查上海天气，翻译成日语，发邮件给info@example.com。",
     "tools":["get_weather","translate","send_email"],"expect":[{"tool":"get_weather","args":{"city":"Shanghai"}},{"tool":"translate","args":{"target_lang":"ja"}},{"tool":"send_email","args":{"to":"info@example.com"}}]},

    # === τ-bench airline / retail (2 cases) ===
    {"id":"a01","cat":"tau_airline","desc":"search flight",
     "en":"I need to fly from Boston to Chicago on August 25.","zh":"我需要8月25日从广州飞北京。",
     "tools":["get_flight"],"expect":[{"tool":"get_flight","args":{"origin":"Boston","destination":"Chicago","date":"2026-08-25"}}]},
    {"id":"r01","cat":"tau_retail","desc":"order lookup + email","ordered":True,
     "en":"Look up order ORD-99887 and email status to me@example.com.","zh":"查订单ORD-99887，发邮件给me@example.com。",
     "tools":["get_order","send_email"],"expect":[{"tool":"get_order","args":{"order_id":"ORD-99887"}},{"tool":"send_email","args":{"to":"me@example.com"}}]},

    # === edge cases (3 cases) ===
    {"id":"e01","cat":"edge","desc":"no tool needed (chitchat)",
     "en":"Hello! How are you today?","zh":"你好！今天怎么样？",
     "tools":["get_weather","calculate"],"expect":[]},
    {"id":"e02","cat":"edge","desc":"ambiguous (no city)",
     "en":"What's the temperature? It's so hot here.","zh":"现在多少度？这儿热死了。",
     "tools":["get_weather"],"expect":[]},
    {"id":"e03","cat":"edge","desc":"mixed EN/ZH params",
     "en":"search news about AI safety, limit 3.","zh":"搜索AI安全新闻，要3条。",
     "tools":["search"],"expect":[{"tool":"search","args":{"query":"AI safety","limit":3}}]},
]


def call_native(model, prompt, tool_names):
    tools = [TOOLS[n] for n in tool_names]
    r = requests.post(f"{OLLAMA}/api/chat", json={
        "model": model, "messages":[{"role":"user","content":prompt}],
        "tools": tools, "stream": False, "options":{"temperature":0}}, timeout=180)
    msg = r.json().get("message", {})
    return [{"tool": tc["function"]["name"], "arguments": tc["function"]["arguments"]}
            for tc in msg.get("tool_calls", [])]

def call_prompt(model, prompt, tool_names):
    descs = []
    for n in tool_names:
        t = TOOLS[n]["function"]; ps = t["parameters"]["properties"]
        req = t["parameters"].get("required",[])
        lines = [f"{k}: {p['type']}{' (required)' if k in req else ''}" for k,p in ps.items()]
        descs.append(f"## {n}\n{t['description']}\n" + "\n".join(f"  {l}" for l in lines))
    full = f"Tools:\n" + "\n\n".join(descs) + f"\n\nOutput JSON array:\n[{{\"tool\":\"name\",\"arguments\":{{}}}}]\n\nUser: {prompt}\nAssistant:"
    r = requests.post(f"{OLLAMA}/api/generate", json={
        "model":model,"prompt":full,"stream":False,"options":{"temperature":0,"num_predict":1024}}, timeout=180)
    text = r.json()["response"].strip()
    text = re.sub(r'^```(?:json)?\s*','',text); text = re.sub(r'\s*```$','',text)
    try:
        d = json.loads(text)
        return d if isinstance(d,list) else ([d] if isinstance(d,dict) else [])
    except:
        m = re.search(r'\[[\s\S]*\]', text)
        return json.loads(m.group()) if m else []

def score_test(expected, actual):
    ne, na = len(expected), len(actual)
    if ne == 0: return 1.0 if na == 0 else 0.0
    if na == 0: return 0.0
    matched = 0
    used = set()
    for exp in expected:
        for j, act in enumerate(actual):
            if j in used: continue
            if exp["tool"] == act.get("tool",""):
                ok = all(str(exp["arguments"].get(k,"")).lower() in str(act.get("arguments",{}).get(k,"")).lower()
                        for k in exp["arguments"])
                if ok: used.add(j); matched += 1; break
    return matched / ne


def run_all():
    models = ["granite4:350m-h","granite4:350m","qwen3:0.6b","qwen3.5:0.8b",
              "lfm2.5-thinking:latest","functiongemma:latest","llama3.2:1b",
              "smollm2:135m","smollm2:360m","smollm2:1.7b","gemma3:1b"]
    has_tools = {"functiongemma:latest","granite4:350m-h","qwen3:0.6b",
                 "granite4:350m","lfm2.5-thinking:latest","qwen3.5:0.8b",
                 "llama3.2:1b","smollm2:1.7b"}

    for model in models:
        fn = call_native if model in has_tools else call_prompt
        results = []
        for tc in TEST_CASES:
            for lang, prompt in [("EN",tc["en"]),("ZH",tc["zh"])]:
                try:
                    calls = fn(model, prompt, tc["tools"])
                    score = score_test(tc["expect"], calls)
                except Exception as e:
                    score = 0.0
                results.append({"id":f"{tc['id']}_{lang}","cat":tc["cat"],
                                "lang":lang,"score":score,"actual":calls if 'calls' in dir() else []})
        # 汇总
        en = [r for r in results if r["lang"]=="EN"]
        zh = [r for r in results if r["lang"]=="ZH"]
        en_pass = sum(1 for r in en if r["score"]>=0.5)/len(en)
        zh_pass = sum(1 for r in zh if r["score"]>=0.5)/len(zh)
        print(f"{model}: pass={sum(1 for r in results if r['score']>=0.5)}/{len(results)} "
              f"EN={en_pass:.1%} ZH={zh_pass:.1%}")

if __name__ == "__main__":
    run_all()

conda activate llmbench
export BFCL_PROJECT_ROOT=$HOME/Documents/GitHub/Others/bfcl/berkeley-function-call-leaderboard
export OPENAI_API_KEY=ollama
export OPENAI_BASE_URL=http://localhost:11434/v1

# 为 Ollama 模型注册 handler（已在 model_config.py 中完成）

# 抽样测试：每个模型跑 simple_python + parallel + irrelevance 共 25 个用例
bfcl generate --model ollama-qwen3-0.6b-FC \
  --test-category simple_python,irrelevance,parallel \
  --run-ids --num-threads 5

bfcl evaluate --model ollama-qwen3-0.6b-FC \
  --test-category simple_python,irrelevance,parallel --partial-eval