here.

\n

Available models:

\n

\n
• https://huggingface.co/epfl-llm/meditron-7b
\n
• https://huggingface.co/epfl-llm/meditron-70B
\n

\n","updatedAt":"2023-11-28T09:45:25.224Z","author":{"_id":"5e6a3d4ea9afd5125d9ec064","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1584020801691-noauth.jpeg","fullname":"Stefan Schweter","name":"stefan-it","type":"user","isPro":true,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":3685,"isUserFollowing":false}},"numEdits":1,"identifiedLanguage":{"language":"en","probability":0.6906628608703613},"editors":["stefan-it"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/1584020801691-noauth.jpeg"],"reactions":[],"isReport":false}}],"primaryEmailConfirmed":false,"paper":{"id":"2311.16079","authors":[{"_id":"6565b230ba64f727ddbee00e","user":{"_id":"654b92086a49f6f6e0edbcd9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/654b92086a49f6f6e0edbcd9/1bWswk7EcL04mWwgx1TpA.jpeg","isPro":true,"fullname":"Zeming Chen - Eric","user":"zechen-nlp","type":"user"},"name":"Zeming Chen","status":"claimed_verified","statusLastChangedAt":"2023-11-29T14:54:56.373Z","hidden":false},{"_id":"6565b230ba64f727ddbee00f","user":{"_id":"648211d39c44e3422ca7e4f3","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/648211d39c44e3422ca7e4f3/GtxJQRx0kjSiNnZ-wXrJ7.png","isPro":false,"fullname":"Alejandro Hernández Cano","user":"alehc","type":"user"},"name":"Alejandro Hernández Cano","status":"claimed_verified","statusLastChangedAt":"2023-12-08T10:20:02.901Z","hidden":false},{"_id":"6565b230ba64f727ddbee010","user":{"_id":"62a0803e3e7d1dda046d475a","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/62a0803e3e7d1dda046d475a/Oi1qHSaX7_PU1pS8qaXKv.jpeg","isPro":false,"fullname":"Angelika Romanou","user":"angelika","type":"user"},"name":"Angelika Romanou","status":"claimed_verified","statusLastChangedAt":"2024-12-02T09:05:56.077Z","hidden":false},{"_id":"6565b230ba64f727ddbee011","user":{"_id":"64bc12cf140491ca9f341c8f","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64bc12cf140491ca9f341c8f/MZ3vilsnnO5jSeao4QNd-.jpeg","isPro":false,"fullname":"Antoine Bonnet","user":"AGBonnet","type":"user"},"name":"Antoine Bonnet","status":"claimed_verified","statusLastChangedAt":"2023-12-04T08:03:07.555Z","hidden":false},{"_id":"6565b230ba64f727ddbee012","name":"Kyle Matoba","hidden":false},{"_id":"6565b230ba64f727ddbee013","user":{"_id":"652c25ac2ecb5062d6c993bc","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/652c25ac2ecb5062d6c993bc/DIZCtdyPBHVR9fwCNFSmC.jpeg","isPro":false,"fullname":"Francesco Salvi","user":"frasalvi","type":"user"},"name":"Francesco Salvi","status":"claimed_verified","statusLastChangedAt":"2023-11-29T14:54:59.987Z","hidden":false},{"_id":"6565b230ba64f727ddbee014","name":"Matteo Pagliardini","hidden":false},{"_id":"6565b230ba64f727ddbee015","user":{"_id":"60d3d1f16a42871838fdcfb3","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60d3d1f16a42871838fdcfb3/B6CqCaGT6Oq7s5txVOomg.jpeg","isPro":false,"fullname":"Simin Fan","user":"Olivia-umich","type":"user"},"name":"Simin Fan","status":"claimed_verified","statusLastChangedAt":"2024-03-14T10:07:00.343Z","hidden":false},{"_id":"6565b230ba64f727ddbee016","user":{"_id":"63b888106cf60e8c3d00dc86","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/63b888106cf60e8c3d00dc86/lNCiIc424q60PC2D33vxN.jpeg","isPro":true,"fullname":"Andreas Köpf","user":"andreaskoepf","type":"user"},"name":"Andreas Köpf","status":"claimed_verified","statusLastChangedAt":"2026-01-15T15:10:03.719Z","hidden":false},{"_id":"6565b230ba64f727ddbee017","name":"Amirkeivan Mohtashami","hidden":false},{"_id":"6565b230ba64f727ddbee018","name":"Alexandre Sallinen","hidden":false},{"_id":"6565b230ba64f727ddbee019","name":"Alireza Sakhaeirad","hidden":false},{"_id":"6565b230ba64f727ddbee01a","name":"Vinitra Swamy","hidden":false},{"_id":"6565b230ba64f727ddbee01b","name":"Igor Krawczuk","hidden":false},{"_id":"6565b230ba64f727ddbee01c","name":"Deniz Bayazit","hidden":false},{"_id":"6565b230ba64f727ddbee01d","name":"Axel Marmet","hidden":false},{"_id":"6565b230ba64f727ddbee01e","name":"Syrielle Montariol","hidden":false},{"_id":"6565b230ba64f727ddbee01f","name":"Mary-Anne Hartley","hidden":false},{"_id":"6565b230ba64f727ddbee020","name":"Martin Jaggi","hidden":false},{"_id":"6565b230ba64f727ddbee021","user":{"_id":"654fb1c67490049d621516fb","avatarUrl":"/avatars/0f88177da236b1bce4a4a58fd0246884.svg","isPro":false,"fullname":"Antoine Bosselut","user":"atcbosselut","type":"user"},"name":"Antoine Bosselut","status":"claimed_verified","statusLastChangedAt":"2024-12-02T08:58:36.288Z","hidden":false}],"publishedAt":"2023-11-27T18:49:43.000Z","title":"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models","summary":"Large language models (LLMs) can potentially democratize access to medical\nknowledge. While many efforts have been made to harness and improve LLMs'\nmedical knowledge and reasoning capacities, the resulting models are either\nclosed-source (e.g., PaLM, GPT-4) or limited in scale (<= 13B parameters),\nwhich restricts their abilities. In this work, we improve access to large-scale\nmedical LLMs by releasing MEDITRON: a suite of open-source LLMs with 7B and 70B\nparameters adapted to the medical domain. MEDITRON builds on Llama-2 (through\nour adaptation of Nvidia's Megatron-LM distributed trainer), and extends\npretraining on a comprehensively curated medical corpus, including selected\nPubMed articles, abstracts, and internationally-recognized medical guidelines.\nEvaluations using four major medical benchmarks show significant performance\ngains over several state-of-the-art baselines before and after task-specific\nfinetuning. Overall, MEDITRON achieves a 6% absolute performance gain over the\nbest public baseline in its parameter class and 3% over the strongest baseline\nwe finetuned from Llama-2. Compared to closed-source LLMs, MEDITRON-70B\noutperforms GPT-3.5 and Med-PaLM and is within 5% of GPT-4 and 10% of\nMed-PaLM-2. We release our code for curating the medical pretraining corpus and\nthe MEDITRON model weights to drive open-source development of more capable\nmedical LLMs.","upvotes":19,"discussionId":"6565b230ba64f727ddbee03a","githubRepo":"https://github.com/epfllm/meditron","githubRepoAddedBy":"auto","ai_summary":"MEDITRON, a suite of open-source LLMs for medical applications, achieves significant performance gains across benchmarks compared to both closed-source and smaller-scale models.","ai_keywords":["large language models","LLMs","medical knowledge","reasoning capacities","open-source","Llama-2","Nvidia's Megatron-LM","distributed trainer","medical corpus","PubMed articles","medical guidelines","medical benchmarks","task-specific finetuning","MEDITRON-70B","GPT-3.5","Med-PaLM","GPT-4","Med-PaLM-2","code curating","model weights"],"githubStars":2145},"canReadDatabase":false,"canManagePapers":false,"canSubmit":false,"hasHfLevelAccess":false,"upvoted":false,"upvoters":[{"_id":"5e6a3d4ea9afd5125d9ec064","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1584020801691-noauth.jpeg","isPro":true,"fullname":"Stefan Schweter","user":"stefan-it","type":"user"},{"_id":"6032802e1f993496bc14d9e3","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6032802e1f993496bc14d9e3/w6hr-DEQot4VVkoyRIBiy.png","isPro":false,"fullname":"Omar Sanseviero","user":"osanseviero","type":"user"},{"_id":"5fcc1929563427b03e9af259","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1620642852901-5fcc1929563427b03e9af259.jpeg","isPro":false,"fullname":"David Adelani","user":"Davlan","type":"user"},{"_id":"5f0c746619cb630495b814fd","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1594651707950-noauth.jpeg","isPro":true,"fullname":"Lewis Tunstall","user":"lewtun","type":"user"},{"_id":"6201c0129dab2e6e083d023c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1656519120848-6201c0129dab2e6e083d023c.jpeg","isPro":false,"fullname":"Katie Link","user":"katielink","type":"user"},{"_id":"652c25ac2ecb5062d6c993bc","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/652c25ac2ecb5062d6c993bc/DIZCtdyPBHVR9fwCNFSmC.jpeg","isPro":false,"fullname":"Francesco Salvi","user":"frasalvi","type":"user"},{"_id":"654a3b0be4ce07c7d77ca12f","avatarUrl":"/avatars/1cde762b02e81910899fe14471696c0c.svg","isPro":false,"fullname":"ANDRE COUTINHO CASTILLA","user":"castillamed","type":"user"},{"_id":"5e845f5d92fa4e3c26ddb32c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/no-auth/_1LS-K5wSSDGwslAVQ3uF.png","isPro":false,"fullname":"Praise","user":"Praise2112","type":"user"},{"_id":"6281d941eeb15579946ca3ce","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6281d941eeb15579946ca3ce/0CdrBop_kjRkOqxUTYFbf.jpeg","isPro":false,"fullname":"Hui Sun","user":"CocoSun","type":"user"},{"_id":"64747f7e33192631bacd8831","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/64747f7e33192631bacd8831/dstkZJ4sHJSeqLesV5cOC.jpeg","isPro":false,"fullname":"Taufiq Dwi Purnomo","user":"taufiqdp","type":"user"},{"_id":"64778e4b8ab7e732b6d7bb41","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/noauth/wprgE8gGM-fWJGXWOI92D.jpeg","isPro":false,"fullname":"siwei liu","user":"TedSiwei","type":"user"},{"_id":"654b92086a49f6f6e0edbcd9","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/654b92086a49f6f6e0edbcd9/1bWswk7EcL04mWwgx1TpA.jpeg","isPro":true,"fullname":"Zeming Chen - Eric","user":"zechen-nlp","type":"user"}],"acceptLanguages":["*"]}">