https://github.com/apple/ToolSandbox

\n","updatedAt":"2024-08-12T03:10:26.254Z","author":{"_id":"60f1abe7544c2adfd699860c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674929746905-60f1abe7544c2adfd699860c.jpeg","fullname":"AK","name":"akhaliq","type":"user","isPro":false,"isHf":true,"isHfAdmin":false,"isMod":false,"followerCount":9179,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.40525439381599426},"editors":["akhaliq"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/1674929746905-60f1abe7544c2adfd699860c.jpeg"],"reactions":[],"isReport":false}},{"id":"66b989061caaa1d77c203b9f","author":{"_id":"65d9498780bafdfb4b218e77","avatarUrl":"/avatars/b5102ca17cd7d422584c5aaa8021bc86.svg","fullname":"acg ","name":"lihua919","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"isUserFollowing":false},"createdAt":"2024-08-12T04:01:10.000Z","type":"comment","data":{"edited":false,"hidden":false,"latest":{"raw":"URL is invalid？","html":"

URL is invalid？

\n","updatedAt":"2024-08-12T04:01:10.432Z","author":{"_id":"65d9498780bafdfb4b218e77","avatarUrl":"/avatars/b5102ca17cd7d422584c5aaa8021bc86.svg","fullname":"acg ","name":"lihua919","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"ja","probability":0.3824819326400757},"editors":["lihua919"],"editorAvatarUrls":["/avatars/b5102ca17cd7d422584c5aaa8021bc86.svg"],"reactions":[{"reaction":"➕","users":["ndurner","promptam","michaelxing","ggianotti"],"count":4}],"isReport":false}},{"id":"66b9ee8781549c6efd27ff7c","author":{"_id":"63d3ea4b640bb0f77176be5d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/M4xSIYXil-CB29A0631Th.png","fullname":"Piyush Sharma","name":"piyusharma","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":2,"isUserFollowing":false},"createdAt":"2024-08-12T11:14:15.000Z","type":"comment","data":{"edited":false,"hidden":false,"latest":{"raw":"@akhaliq seems like GitHub is not available. Please check.","html":"

\n\n@akhaliq\n\t seems like GitHub is not available. Please check.

\n","updatedAt":"2024-08-12T11:14:15.343Z","author":{"_id":"63d3ea4b640bb0f77176be5d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/M4xSIYXil-CB29A0631Th.png","fullname":"Piyush Sharma","name":"piyusharma","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":2,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.9576066136360168},"editors":["piyusharma"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/M4xSIYXil-CB29A0631Th.png"],"reactions":[],"isReport":false}},{"id":"66bab7a332be421cd8a6559d","author":{"_id":"63d3e0e8ff1384ce6c5dd17d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg","fullname":"Librarian Bot (Bot)","name":"librarian-bot","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":318,"isUserFollowing":false},"createdAt":"2024-08-13T01:32:19.000Z","type":"comment","data":{"edited":false,"hidden":false,"latest":{"raw":"This is an automated message from the [Librarian Bot](https://huggingface.co/librarian-bots). I found the following papers similar to this paper. \n\nThe following papers were recommended by the Semantic Scholar API \n\n* [τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains](https://huggingface.co/papers/2406.12045) (2024)\n* [Can Tool-augmented Large Language Models be Aware of Incomplete Conditions?](https://huggingface.co/papers/2406.12307) (2024)\n* [MMAU: A Holistic Benchmark of Agent Capabilities Across Diverse Domains](https://huggingface.co/papers/2407.18961) (2024)\n* [Mobile-Bench: An Evaluation Benchmark for LLM-based Mobile Agents](https://huggingface.co/papers/2407.00993) (2024)\n* [AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents](https://huggingface.co/papers/2406.13352) (2024)\n\n\n Please give a thumbs up to this comment if you found it helpful!\n\n If you want recommendations for any Paper on Hugging Face checkout [this](https://huggingface.co/spaces/librarian-bots/recommend_similar_papers) Space\n\n You can directly ask Librarian Bot for paper recommendations by tagging it in a comment: `@librarian-bot recommend`","html":"

This is an automated message from the Librarian Bot. I found the following papers similar to this paper.

\n

The following papers were recommended by the Semantic Scholar API

\n

\n
• τ-bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains (2024)
\n
• Can Tool-augmented Large Language Models be Aware of Incomplete Conditions? (2024)
\n
• MMAU: A Holistic Benchmark of Agent Capabilities Across Diverse Domains (2024)
\n
• Mobile-Bench: An Evaluation Benchmark for LLM-based Mobile Agents (2024)
\n
• AgentDojo: A Dynamic Environment to Evaluate Attacks and Defenses for LLM Agents (2024)
\n

\n

Please give a thumbs up to this comment if you found it helpful!

\n

If you want recommendations for any Paper on Hugging Face checkout this Space

\n

You can directly ask Librarian Bot for paper recommendations by tagging it in a comment: \n\n@librarian-bot\n\t recommend

\n","updatedAt":"2024-08-13T01:32:19.545Z","author":{"_id":"63d3e0e8ff1384ce6c5dd17d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg","fullname":"Librarian Bot (Bot)","name":"librarian-bot","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":318,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.7238712310791016},"editors":["librarian-bot"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg"],"reactions":[],"isReport":false}}],"primaryEmailConfirmed":false,"paper":{"id":"2408.04682","authors":[{"_id":"66b97d1874d4fb4d570ee3c7","user":{"_id":"64bad07938953777fe3ed5e9","avatarUrl":"/avatars/91c07a176d8d51111873751e82a4e0ea.svg","isPro":false,"fullname":"Jiarui Lu","user":"jiarui-lu","type":"user"},"name":"Jiarui Lu","status":"admin_assigned","statusLastChangedAt":"2024-08-13T11:16:26.743Z","hidden":false},{"_id":"66b97d1874d4fb4d570ee3c8","name":"Thomas Holleis","hidden":false},{"_id":"66b97d1874d4fb4d570ee3c9","user":{"_id":"64d1284c59503263d91b4707","avatarUrl":"/avatars/12d3799af150b538ad87874fdbf49dd1.svg","isPro":false,"fullname":"Yizhe Zhang","user":"yizheapple","type":"user"},"name":"Yizhe Zhang","status":"admin_assigned","statusLastChangedAt":"2024-08-13T11:16:52.951Z","hidden":false},{"_id":"66b97d1874d4fb4d570ee3ca","name":"Bernhard Aumayer","hidden":false},{"_id":"66b97d1874d4fb4d570ee3cb","name":"Feng Nan","hidden":false},{"_id":"66b97d1874d4fb4d570ee3cc","user":{"_id":"6468582ba9b4610868a654ab","avatarUrl":"/avatars/1f15b922a45e43e24905f9df98346ca5.svg","isPro":false,"fullname":"Felix Bai","user":"fbai","type":"user"},"name":"Felix Bai","status":"admin_assigned","statusLastChangedAt":"2024-08-13T11:17:20.075Z","hidden":false},{"_id":"66b97d1874d4fb4d570ee3cd","name":"Shuang Ma","hidden":false},{"_id":"66b97d1874d4fb4d570ee3ce","name":"Shen Ma","hidden":false},{"_id":"66b97d1874d4fb4d570ee3cf","user":{"_id":"6530b8bdcb5ad115c394e7cd","avatarUrl":"/avatars/5621f2394e6c382c09147185b56d4154.svg","isPro":false,"fullname":"Mengyu Li","user":"molee1004","type":"user"},"name":"Mengyu Li","status":"admin_assigned","statusLastChangedAt":"2024-08-13T11:17:44.954Z","hidden":false},{"_id":"66b97d1874d4fb4d570ee3d0","user":{"_id":"605b7f42935268bc086131ba","avatarUrl":"/avatars/a55109f714b33f9d59d69011ddeb0b9f.svg","isPro":false,"fullname":"Guoli Yin","user":"gyin94","type":"user"},"name":"Guoli Yin","status":"admin_assigned","statusLastChangedAt":"2024-08-13T11:17:50.580Z","hidden":false},{"_id":"66b97d1874d4fb4d570ee3d1","name":"Zirui Wang","hidden":false},{"_id":"66b97d1874d4fb4d570ee3d2","user":{"_id":"654ef8ad3fe6c0b1f871942f","avatarUrl":"/avatars/8d3689b9bf57c7c8060ac510a1839158.svg","isPro":false,"fullname":"Ruoming Pang","user":"ruoming","type":"user"},"name":"Ruoming Pang","status":"admin_assigned","statusLastChangedAt":"2024-08-13T11:18:19.204Z","hidden":false}],"publishedAt":"2024-08-08T05:45:42.000Z","submittedOnDailyAt":"2024-08-12T01:40:26.235Z","title":"ToolSandbox: A Stateful, Conversational, Interactive Evaluation\n Benchmark for LLM Tool Use Capabilities","submittedOnDailyBy":{"_id":"60f1abe7544c2adfd699860c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674929746905-60f1abe7544c2adfd699860c.jpeg","isPro":false,"fullname":"AK","user":"akhaliq","type":"user"},"summary":"Recent large language models (LLMs) advancements sparked a growing research\ninterest in tool assisted LLMs solving real-world challenges, which calls for\ncomprehensive evaluation of tool-use capabilities. While previous works focused\non either evaluating over stateless web services (RESTful API), based on a\nsingle turn user prompt, or an off-policy dialog trajectory, ToolSandbox\nincludes stateful tool execution, implicit state dependencies between tools, a\nbuilt-in user simulator supporting on-policy conversational evaluation and a\ndynamic evaluation strategy for intermediate and final milestones over an\narbitrary trajectory. We show that open source and proprietary models have a\nsignificant performance gap, and complex tasks like State Dependency,\nCanonicalization and Insufficient Information defined in ToolSandbox are\nchallenging even the most capable SOTA LLMs, providing brand-new insights into\ntool-use LLM capabilities. ToolSandbox evaluation framework is released at\nhttps://github.com/apple/ToolSandbox","upvotes":18,"discussionId":"66b97d1c74d4fb4d570ee4ec","githubRepo":"https://github.com/apple/toolsandbox","githubRepoAddedBy":"auto","ai_summary":"ToolSandbox evaluates large language models with tool-use capabilities through stateful execution, implicit state dependencies, and conversational simulation, revealing performance gaps and challenges with complex tasks.","ai_keywords":["stateful tool execution","implicit state dependencies","user simulator","on-policy evaluation","conversational evaluation","dynamic evaluation strategy","state dependency","canonicalization","insufficient information"],"githubStars":238},"canReadDatabase":false,"canManagePapers":false,"canSubmit":false,"hasHfLevelAccess":false,"upvoted":false,"upvoters":[{"_id":"620783f24e28382272337ba4","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/620783f24e28382272337ba4/zkUveQPNiDfYjgGhuFErj.jpeg","isPro":false,"fullname":"GuoLiangTang","user":"Tommy930","type":"user"},{"_id":"648f4ced9ae7cc4fcf02a28c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/648f4ced9ae7cc4fcf02a28c/v_M8M9WfrTCcg_pvccwor.jpeg","isPro":true,"fullname":"Nils Durner","user":"ndurner","type":"user"},{"_id":"61b839889f7cfeae618e72c9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/61b839889f7cfeae618e72c9/5kFRCChdqwv7MGM8T_y5v.jpeg","isPro":false,"fullname":"Raja Biswas","user":"rbiswasfc","type":"user"},{"_id":"6454fca5a13edf669cd75b83","avatarUrl":"/avatars/0b8b529410da8e0df44c3da3618c2991.svg","isPro":false,"fullname":"Vincent","user":"promptam","type":"user"},{"_id":"62716952bcef985363db8485","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/62716952bcef985363db8485/zJPPo5xlwZRJdEuwYsYKp.jpeg","isPro":true,"fullname":"JB D.","user":"IAMJB","type":"user"},{"_id":"6270324ebecab9e2dcf245de","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6270324ebecab9e2dcf245de/cMbtWSasyNlYc9hvsEEzt.jpeg","isPro":false,"fullname":"Kye Gomez","user":"kye","type":"user"},{"_id":"651c80a26ba9ab9b9582c273","avatarUrl":"/avatars/e963452eafd21f517d800f2e58e0f918.svg","isPro":false,"fullname":"siyeng feng","user":"siyengfeng","type":"user"},{"_id":"60078446e55258e41786a959","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60078446e55258e41786a959/UGPCE4YqG9BVMSf0YauxL.png","isPro":false,"fullname":"Motoki Wu","user":"tokestermw","type":"user"},{"_id":"64587be872b60ae7a3817858","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64587be872b60ae7a3817858/BbdOOxOCEzWTvEpkWp8MM.png","isPro":false,"fullname":"Minbyul Jeong","user":"Minbyul","type":"user"},{"_id":"641b754d1911d3be6745cce9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/641b754d1911d3be6745cce9/Ydjcjd4VuNUGj5Cd4QHdB.png","isPro":false,"fullname":"atayloraerospace","user":"Taylor658","type":"user"},{"_id":"64aea8ff67511bd3d965697b","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64aea8ff67511bd3d965697b/Jxn52EmDF5RApJh8antxn.jpeg","isPro":false,"fullname":"Feynman Innovations","user":"ajibawa-2023","type":"user"},{"_id":"646427889dd8b530a8615fd8","avatarUrl":"/avatars/72a38d297cec02cdad7c8555dd0e759f.svg","isPro":false,"fullname":"Vince","user":"bolerovt","type":"user"}],"acceptLanguages":["*"],"dailyPaperRank":0}">