Deprecated: The each() function is deprecated. This message will be suppressed on further calls in /home/zhenxiangba/zhenxiangba.com/public_html/phproxy-improved-master/index.php on line 456 Paper page - GMAI-MMBench: A Comprehensive Multimodal Evaluation Benchmark Towards
General Medical AI
Please give a thumbs up to this comment if you found it helpful!
\n
If you want recommendations for any Paper on Hugging Face checkout this Space
\n
You can directly ask Librarian Bot for paper recommendations by tagging it in a comment: \n\n@librarian-bot\n\t recommend
\n","updatedAt":"2024-08-10T01:33:14.365Z","author":{"_id":"63d3e0e8ff1384ce6c5dd17d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg","fullname":"Librarian Bot (Bot)","name":"librarian-bot","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":318,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.7829102277755737},"editors":["librarian-bot"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg"],"reactions":[],"isReport":false}}],"primaryEmailConfirmed":false,"paper":{"id":"2408.03361","authors":[{"_id":"66b57e63e1f790d2510cd344","name":"Pengcheng Chen","hidden":false},{"_id":"66b57e63e1f790d2510cd345","user":{"_id":"656292f6412dcdebf2c1c6a8","avatarUrl":"/avatars/310604f8c7d701565e9a659e8e85621d.svg","isPro":false,"fullname":"Jin Ye","user":"EugeneYonng","type":"user"},"name":"Jin Ye","status":"claimed_verified","statusLastChangedAt":"2024-11-25T20:39:46.118Z","hidden":false},{"_id":"66b57e63e1f790d2510cd346","user":{"_id":"6409b7eb96aae649740e2920","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1678358480751-noauth.png","isPro":false,"fullname":"Guoan Wang","user":"SolooooMan","type":"user"},"name":"Guoan Wang","status":"claimed_verified","statusLastChangedAt":"2024-08-09T07:47:30.273Z","hidden":false},{"_id":"66b57e63e1f790d2510cd347","name":"Yanjun Li","hidden":false},{"_id":"66b57e63e1f790d2510cd348","name":"Zhongying Deng","hidden":false},{"_id":"66b57e63e1f790d2510cd349","name":"Wei Li","hidden":false},{"_id":"66b57e63e1f790d2510cd34a","user":{"_id":"64aff8ecf808e8d2023804ce","avatarUrl":"/avatars/13d32347626814801c0f9b2d893b6741.svg","isPro":false,"fullname":"li","user":"tianbin","type":"user"},"name":"Tianbin Li","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:34:10.951Z","hidden":false},{"_id":"66b57e63e1f790d2510cd34b","user":{"_id":"63ee1379190ddd6214efd73a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1676546883247-noauth.png","isPro":false,"fullname":"HAODONG DUAN","user":"KennyUTC","type":"user"},"name":"Haodong Duan","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:29:22.932Z","hidden":false},{"_id":"66b57e63e1f790d2510cd34c","user":{"_id":"643fddd6388f31f64ccb9344","avatarUrl":"/avatars/c029b59cfbbc4124489e72fa8e71ee9a.svg","isPro":false,"fullname":"Ziyan Huang","user":"ziyanhuang","type":"user"},"name":"Ziyan Huang","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:29:29.120Z","hidden":false},{"_id":"66b57e63e1f790d2510cd34d","name":"Yanzhou Su","hidden":false},{"_id":"66b57e63e1f790d2510cd34e","user":{"_id":"637c6703ca8542a0ba900ccb","avatarUrl":"/avatars/288ed63a1efa566c3f01e850c6ba5dd5.svg","isPro":false,"fullname":"Wang","user":"Benyou","type":"user"},"name":"Benyou Wang","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:29:45.299Z","hidden":false},{"_id":"66b57e63e1f790d2510cd34f","name":"Shaoting Zhang","hidden":false},{"_id":"66b57e63e1f790d2510cd350","name":"Bin Fu","hidden":false},{"_id":"66b57e63e1f790d2510cd351","user":{"_id":"656fda082f058b368c27d1b9","avatarUrl":"/avatars/cba33f6b20fc93ac9d3d579cbd7f839a.svg","isPro":false,"fullname":"cai jianfei","user":"genify","type":"user"},"name":"Jianfei Cai","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:30:13.163Z","hidden":false},{"_id":"66b57e63e1f790d2510cd352","user":{"_id":"65a8a828ef14f9e6037dfad2","avatarUrl":"/avatars/55fdc0393b2e37ac998ddbdb1e1ff636.svg","isPro":false,"fullname":"Bohan Zhuang","user":"BohanZ","type":"user"},"name":"Bohan Zhuang","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:30:19.255Z","hidden":false},{"_id":"66b57e63e1f790d2510cd353","name":"Eric J Seibel","hidden":false},{"_id":"66b57e63e1f790d2510cd354","user":{"_id":"66b593026cef22e6ba6adb8a","avatarUrl":"/avatars/8a94d55b85177e84d65dd0bd537e335f.svg","isPro":false,"fullname":"JunjunHe","user":"JunjunHe","type":"user"},"name":"Junjun He","status":"admin_assigned","statusLastChangedAt":"2024-08-09T09:30:32.614Z","hidden":false},{"_id":"66b57e63e1f790d2510cd355","name":"Yu Qiao","hidden":false}],"mediaUrls":["https://cdn-uploads.huggingface.co/production/uploads/63ee1379190ddd6214efd73a/DqG8dfM3KjJKBLBf6bqHC.png"],"publishedAt":"2024-08-06T17:59:21.000Z","submittedOnDailyAt":"2024-08-09T01:01:55.456Z","title":"GMAI-MMBench: A Comprehensive Multimodal Evaluation Benchmark Towards\n General Medical AI","submittedOnDailyBy":{"_id":"63ee1379190ddd6214efd73a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1676546883247-noauth.png","isPro":false,"fullname":"HAODONG DUAN","user":"KennyUTC","type":"user"},"summary":"Large Vision-Language Models (LVLMs) are capable of handling diverse data\ntypes such as imaging, text, and physiological signals, and can be applied in\nvarious fields. In the medical field, LVLMs have a high potential to offer\nsubstantial assistance for diagnosis and treatment. Before that, it is crucial\nto develop benchmarks to evaluate LVLMs' effectiveness in various medical\napplications. Current benchmarks are often built upon specific academic\nliterature, mainly focusing on a single domain, and lacking varying perceptual\ngranularities. Thus, they face specific challenges, including limited clinical\nrelevance, incomplete evaluations, and insufficient guidance for interactive\nLVLMs. To address these limitations, we developed the GMAI-MMBench, the most\ncomprehensive general medical AI benchmark with well-categorized data structure\nand multi-perceptual granularity to date. It is constructed from 285 datasets\nacross 39 medical image modalities, 18 clinical-related tasks, 18 departments,\nand 4 perceptual granularities in a Visual Question Answering (VQA) format.\nAdditionally, we implemented a lexical tree structure that allows users to\ncustomize evaluation tasks, accommodating various assessment needs and\nsubstantially supporting medical AI research and applications. We evaluated 50\nLVLMs, and the results show that even the advanced GPT-4o only achieves an\naccuracy of 52%, indicating significant room for improvement. Moreover, we\nidentified five key insufficiencies in current cutting-edge LVLMs that need to\nbe addressed to advance the development of better medical applications. We\nbelieve that GMAI-MMBench will stimulate the community to build the next\ngeneration of LVLMs toward GMAI.\n Project Page: https://uni-medical.github.io/GMAI-MMBench.github.io/","upvotes":85,"discussionId":"66b57e68e1f790d2510cd4c6","githubRepo":"https://github.com/uni-medical/GMAI-MMBench","githubRepoAddedBy":"auto","ai_summary":"GMAI-MMBench is a comprehensive general medical AI benchmark that evaluates large vision-language models across multiple medical domains and perceptual granularities, identifying key insufficiencies and highlighting areas for improvement.","ai_keywords":["Large Vision-Language Models","LVLMs","medical image modalities","clinical-related tasks","perceptual granularities","Visual Question Answering","VQA","lexical tree structure","GPT-4","GMAI"],"githubStars":82},"canReadDatabase":false,"canManagePapers":false,"canSubmit":false,"hasHfLevelAccess":false,"upvoted":false,"upvoters":[{"_id":"63ee1379190ddd6214efd73a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1676546883247-noauth.png","isPro":false,"fullname":"HAODONG DUAN","user":"KennyUTC","type":"user"},{"_id":"65baf87b5d2ef0a4b2f48c73","avatarUrl":"/avatars/a9a75ea3d0c87a2e5b990447943f2394.svg","isPro":false,"fullname":"Zixun Huang","user":"JacksonH0214","type":"user"},{"_id":"648e5897d469965a66b74748","avatarUrl":"/avatars/6eaa371028403ee2b00fab3acd77df42.svg","isPro":false,"fullname":"chasephd","user":"chasephd","type":"user"},{"_id":"643df85c975b365d472b545e","avatarUrl":"/avatars/f6406ebbc8bac11d6157b3b56e7280d1.svg","isPro":false,"fullname":"simzhangbest","user":"simzhang","type":"user"},{"_id":"62b31667cbd2a402fc5e5b0f","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1655903835218-noauth.jpeg","isPro":false,"fullname":"SII-游狸态","user":"RadiCat","type":"user"},{"_id":"65250a3d02387d10b432cab3","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/65250a3d02387d10b432cab3/6pR4d0Hn2mlQ4ROjGT2P0.png","isPro":false,"fullname":"吴孟松","user":"fairyshine","type":"user"},{"_id":"66a117a3b824c037736ce39a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/cTxQus1FBVbpX8_48iqGv.jpeg","isPro":false,"fullname":"苏德昭","user":"ssdddzzzz","type":"user"},{"_id":"6694995f3cd89359e448739b","avatarUrl":"/avatars/d2db7b2d35eed04eb878eb63dec2c780.svg","isPro":false,"fullname":"Kelly","user":"Haozy","type":"user"},{"_id":"66440d89ddafd0457dfea702","avatarUrl":"/avatars/b0d111ec15143d2c1f1f9263c47db44c.svg","isPro":false,"fullname":"gaoyu","user":"yugao94","type":"user"},{"_id":"643fddd6388f31f64ccb9344","avatarUrl":"/avatars/c029b59cfbbc4124489e72fa8e71ee9a.svg","isPro":false,"fullname":"Ziyan Huang","user":"ziyanhuang","type":"user"},{"_id":"6281d941eeb15579946ca3ce","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6281d941eeb15579946ca3ce/0CdrBop_kjRkOqxUTYFbf.jpeg","isPro":false,"fullname":"Hui Sun","user":"CocoSun","type":"user"},{"_id":"6597b594a0f237a28142e208","avatarUrl":"/avatars/39831ef3b96db0be9daec694a490bf46.svg","isPro":false,"fullname":"zhangchangqing","user":"Facea1l","type":"user"}],"acceptLanguages":["*"],"dailyPaperRank":2}">
GMAI-MMBench is a comprehensive general medical AI benchmark that evaluates large vision-language models across multiple medical domains and perceptual granularities, identifying key insufficiencies and highlighting areas for improvement.
AI-generated summary
Large Vision-Language Models (LVLMs) are capable of handling diverse data
types such as imaging, text, and physiological signals, and can be applied in
various fields. In the medical field, LVLMs have a high potential to offer
substantial assistance for diagnosis and treatment. Before that, it is crucial
to develop benchmarks to evaluate LVLMs' effectiveness in various medical
applications. Current benchmarks are often built upon specific academic
literature, mainly focusing on a single domain, and lacking varying perceptual
granularities. Thus, they face specific challenges, including limited clinical
relevance, incomplete evaluations, and insufficient guidance for interactive
LVLMs. To address these limitations, we developed the GMAI-MMBench, the most
comprehensive general medical AI benchmark with well-categorized data structure
and multi-perceptual granularity to date. It is constructed from 285 datasets
across 39 medical image modalities, 18 clinical-related tasks, 18 departments,
and 4 perceptual granularities in a Visual Question Answering (VQA) format.
Additionally, we implemented a lexical tree structure that allows users to
customize evaluation tasks, accommodating various assessment needs and
substantially supporting medical AI research and applications. We evaluated 50
LVLMs, and the results show that even the advanced GPT-4o only achieves an
accuracy of 52%, indicating significant room for improvement. Moreover, we
identified five key insufficiencies in current cutting-edge LVLMs that need to
be addressed to advance the development of better medical applications. We
believe that GMAI-MMBench will stimulate the community to build the next
generation of LVLMs toward GMAI.
Project Page: https://uni-medical.github.io/GMAI-MMBench.github.io/
A comprehensive benchmark for medical image understanding, encompassing 39 medical image modalities, 18 clinical-related tasks, 18 departments, and 4 perceptual granularities in the VQA format. The benchmark provides extensive evaluation results of 50 LVLMs, including GPT-4o, Gemini-1.5, and others.