Hugging Quants

\n

Welcome to the home of exciting quantized models! We'd love to see increased adoption of powerful state-of-the-art open models, and quantization is a key component to make them work on more types of hardware.

\n

Resources:

\n

\n
• Llama 3.1 Quantized Models: Optimised Quants of Llama 3.1 for high-throughput deployments! Compatible with Transformers, TGI & VLLM 🤗.
\n
• Hugging Face Llama Recipes: A set of minimal recipes to get started with Llama 3.1.
\n

\n","classNames":"hf-sanitized hf-sanitized-z5XdL9yAlxnkOJzOp4OLV"},"users":[{"_id":"60f0608166e5701b80ed3f02","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/x3tcqufwDX_d0N69VVNvn.jpeg","isPro":false,"fullname":"Alvaro Bartolome","user":"alvarobartt","type":"user"},{"_id":"6310e9a107a76827902421d7","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6310e9a107a76827902421d7/1e0GItqv3A6sOWYz5SXoe.png","isPro":false,"fullname":"Daniël de Kok","user":"danieldk","type":"user"},{"_id":"63ce875d199b36f7552d4f07","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/63ce875d199b36f7552d4f07/bpUrvhXDagzRqZ3vxTcSF.jpeg","isPro":false,"fullname":"Marc Sun","user":"marcsun13","type":"user"},{"_id":"6400f6cc2b67d27affcfdb93","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6400f6cc2b67d27affcfdb93/WA6FEZy_YaZPGhIWj2zda.jpeg","isPro":true,"fullname":"Matthew Douglas","user":"mdouglas","type":"user"},{"_id":"61b253b7ac5ecaae3d1efe0c","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/61b253b7ac5ecaae3d1efe0c/hwiQ0uvz3t-L5a-NtBIO6.png","isPro":false,"fullname":"Joshua","user":"Xenova","type":"user"},{"_id":"603d25b75f9d390ab190b777","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1617264212503-603d25b75f9d390ab190b777.jpeg","isPro":true,"fullname":"Pedro Cuenca","user":"pcuenq","type":"user"},{"_id":"6632d7e22c4f4bfc3f6a05c2","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6632d7e22c4f4bfc3f6a05c2/TCDlrb-O5aormNjSX-tyE.png","isPro":false,"fullname":"Mohamed Mekkouri","user":"medmekk","type":"user"}],"userCount":7,"collections":[{"slug":"hugging-quants/gemma2-awq-quants-6710cb8caa5c3585cb49ad24","title":"Gemma2 AWQ Quants","description":"Optimised AWQ Quants for high-throughput deployments of Gemma2! Compatible with Transformers, TGI & VLLM 🤗","gating":false,"lastUpdated":"2024-10-17T08:32:58.045Z","owner":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"items":[{"_id":"6710cbba2f2350276f3d7d4c","position":0,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":2457,"gated":false,"id":"hugging-quants/gemma-2-9b-it-AWQ-INT4","availableInferenceProviders":[],"lastModified":"2024-10-17T08:31:37.000Z","likes":8,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":9241705984}],"position":0,"theme":"orange","private":false,"shareUrl":"https://hf.co/collections/hugging-quants/gemma2-awq-quants","upvotes":0,"isUpvotedByUser":false},{"slug":"hugging-quants/llama-32-3b-and-1b-gguf-quants-66f43204a559009763c009a5","title":"Llama 3.2 3B & 1B GGUF Quants","description":"Llama.cpp compatible quants for Llama 3.2 3B and 1B Instruct models.","gating":false,"lastUpdated":"2024-09-26T09:27:24.772Z","owner":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"items":[{"_id":"66f4323a4bfac05e4561af58","position":0,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":5206,"gated":false,"id":"hugging-quants/Llama-3.2-3B-Instruct-Q8_0-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:11:19.000Z","likes":52,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":3212749888},{"_id":"66f43251ed02030ede899add","position":1,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":18845,"gated":false,"id":"hugging-quants/Llama-3.2-3B-Instruct-Q4_K_M-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:12:08.000Z","likes":25,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":3212749888},{"_id":"66f4324758df8addbc6ad7b1","position":2,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":465568,"gated":false,"id":"hugging-quants/Llama-3.2-1B-Instruct-Q8_0-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:14:40.000Z","likes":43,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":1235814432},{"_id":"66f4325a2d7c50d9635a9ff5","position":3,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":35324,"gated":false,"id":"hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:15:26.000Z","likes":19,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":1235814432}],"position":1,"theme":"pink","private":false,"shareUrl":"https://hf.co/collections/hugging-quants/llama-32-3b-and-1b-gguf-quants","upvotes":46,"isUpvotedByUser":false},{"slug":"hugging-quants/llama-31-gptq-awq-and-bnb-quants-669fa7f50f6e713fd54bd198","title":"Llama 3.1 GPTQ, AWQ, and BNB Quants","description":"Optimised Quants for high-throughput deployments! Compatible with Transformers, TGI & VLLM 🤗","gating":false,"lastUpdated":"2024-09-26T09:27:24.764Z","owner":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"items":[{"_id":"669fa8703b600571748e470f","position":0,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":930,"gated":false,"id":"hugging-quants/Meta-Llama-3.1-405B-Instruct-AWQ-INT4","availableInferenceProviders":[],"lastModified":"2024-09-13T06:37:37.000Z","likes":36,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":410081247232},{"_id":"66a0ee67c8be189aea2ba207","position":1,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":10,"gated":false,"id":"hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4","availableInferenceProviders":[],"lastModified":"2024-09-16T23:46:54.000Z","likes":5,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":422790142440},{"_id":"669fa87993fdcfcb30ad4dd5","position":2,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":62,"gated":false,"id":"hugging-quants/Meta-Llama-3.1-405B-Instruct-GPTQ-INT4","availableInferenceProviders":[],"lastModified":"2024-08-07T07:28:07.000Z","likes":16,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":410101374976},{"_id":"669fa886ea38f3fcba4091fb","position":4,"type":"model","author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":92273,"gated":false,"id":"hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4","availableInferenceProviders":[],"lastModified":"2024-08-07T07:16:54.000Z","likes":107,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[]}],"position":2,"theme":"orange","private":false,"shareUrl":"https://hf.co/collections/hugging-quants/llama-31-gptq-awq-and-bnb-quants","upvotes":57,"isUpvotedByUser":false}],"datasets":[],"models":[{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":0,"gated":false,"id":"hugging-quants/Llama-4-Scout-17B-16E-Instruct-fbgemm","availableInferenceProviders":[],"lastModified":"2025-04-09T10:57:10.000Z","likes":2,"pipeline_tag":"image-text-to-text","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":108659931392},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":1,"gated":false,"id":"hugging-quants/Llama-4-Scout-17B-16E-Instruct-fbgemm-unfused","availableInferenceProviders":[],"lastModified":"2025-04-09T08:57:23.000Z","likes":2,"pipeline_tag":"image-text-to-text","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":108659931392},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":2457,"gated":false,"id":"hugging-quants/gemma-2-9b-it-AWQ-INT4","availableInferenceProviders":[],"lastModified":"2024-10-17T08:31:37.000Z","likes":8,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":9241705984},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":393,"gated":false,"id":"hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4","availableInferenceProviders":[],"lastModified":"2024-10-07T06:59:00.000Z","likes":0,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"numParameters":46702792704},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":35324,"gated":false,"id":"hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:15:26.000Z","likes":19,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":1235814432},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":465568,"gated":false,"id":"hugging-quants/Llama-3.2-1B-Instruct-Q8_0-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:14:40.000Z","likes":43,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":1235814432},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":18845,"gated":false,"id":"hugging-quants/Llama-3.2-3B-Instruct-Q4_K_M-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:12:08.000Z","likes":25,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":3212749888},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":5206,"gated":false,"id":"hugging-quants/Llama-3.2-3B-Instruct-Q8_0-GGUF","availableInferenceProviders":[],"lastModified":"2024-09-25T16:11:19.000Z","likes":52,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":3212749888},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":9,"gated":false,"id":"hugging-quants/Meta-Llama-3.1-405B-BNB-NF4","availableInferenceProviders":[],"lastModified":"2024-09-16T23:55:41.000Z","likes":2,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":418429905348},{"author":"hugging-quants","authorData":{"_id":"66978677ebf473b4a901c289","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/60f0608166e5701b80ed3f02/Mm_zQGapvXKw4VCCJ8Jbc.png","fullname":"Hugging Quants","name":"hugging-quants","type":"org","isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":275,"isUserFollowing":false},"downloads":10,"gated":false,"id":"hugging-quants/Meta-Llama-3.1-405B-Instruct-BNB-NF4","availableInferenceProviders":[],"lastModified":"2024-09-16T23:46:54.000Z","likes":5,"pipeline_tag":"text-generation","private":false,"repoType":"model","isLikedByUser":false,"widgetOutputUrls":[],"numParameters":422790142440}],"paperPreviews":[],"spaces":[],"buckets":[],"numBuckets":0,"numDatasets":0,"numModels":21,"numSpaces":1,"lastOrgActivities":[],"acceptLanguages":["*"],"canReadRepos":false,"canReadSpaces":false,"blogPosts":[],"currentRepoPage":0,"filters":{},"paperView":false}">