https://lyra-omni.github.io/
Demo: https://103.170.5.190:17860/
Code: https://github.com/dvlab-research/Lyra
Model: https://huggingface.co/collections/zszhong/lyra-model-674ea5bb3b39ff8f15de75fc

\n","updatedAt":"2024-12-13T05:58:45.203Z","author":{"_id":"6423e35b30b0e4ab36dd1b16","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6423e35b30b0e4ab36dd1b16/pea6LVDS9PQAxvQt9GUiZ.jpeg","fullname":"Wang Chengyao","name":"wcy1122","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":41,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.7920132875442505},"editors":["wcy1122"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/6423e35b30b0e4ab36dd1b16/pea6LVDS9PQAxvQt9GUiZ.jpeg"],"reactions":[{"reaction":"❤️","users":["julianjuaner","Yukkkop","oceansweep","wcy1122","ShuLiu","zszhong"],"count":6}],"isReport":false}},{"id":"675ce0ebc0c290bb7a7d9c97","author":{"_id":"63d3e0e8ff1384ce6c5dd17d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg","fullname":"Librarian Bot (Bot)","name":"librarian-bot","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":318,"isUserFollowing":false},"createdAt":"2024-12-14T01:35:39.000Z","type":"comment","data":{"edited":false,"hidden":false,"latest":{"raw":"This is an automated message from the [Librarian Bot](https://huggingface.co/librarian-bots). I found the following papers similar to this paper. \n\nThe following papers were recommended by the Semantic Scholar API \n\n* [LongVALE: Vision-Audio-Language-Event Benchmark Towards Time-Aware Omni-Modal Perception of Long Videos](https://huggingface.co/papers/2411.19772) (2024)\n* [Ichigo: Mixed-Modal Early-Fusion Realtime Voice Assistant](https://huggingface.co/papers/2410.15316) (2024)\n* [Towards Multi-Modal Mastery: A 4.5B Parameter Truly Multi-Modal Small Language Model](https://huggingface.co/papers/2411.05903) (2024)\n* [Get Large Language Models Ready to Speak: A Late-fusion Approach for Speech Generation](https://huggingface.co/papers/2410.20336) (2024)\n* [Improving Multi-modal Large Language Model through Boosting Vision Capabilities](https://huggingface.co/papers/2410.13733) (2024)\n* [Spider: Any-to-Many Multimodal LLM](https://huggingface.co/papers/2411.09439) (2024)\n* [Continuous Speech Tokens Makes LLMs Robust Multi-Modality Learners](https://huggingface.co/papers/2412.04917) (2024)\n\n\n Please give a thumbs up to this comment if you found it helpful!\n\n If you want recommendations for any Paper on Hugging Face checkout [this](https://huggingface.co/spaces/librarian-bots/recommend_similar_papers) Space\n\n You can directly ask Librarian Bot for paper recommendations by tagging it in a comment: `@librarian-bot recommend`","html":"

This is an automated message from the Librarian Bot. I found the following papers similar to this paper.

\n

The following papers were recommended by the Semantic Scholar API

\n

\n
• LongVALE: Vision-Audio-Language-Event Benchmark Towards Time-Aware Omni-Modal Perception of Long Videos (2024)
\n
• Ichigo: Mixed-Modal Early-Fusion Realtime Voice Assistant (2024)
\n
• Towards Multi-Modal Mastery: A 4.5B Parameter Truly Multi-Modal Small Language Model (2024)
\n
• Get Large Language Models Ready to Speak: A Late-fusion Approach for Speech Generation (2024)
\n
• Improving Multi-modal Large Language Model through Boosting Vision Capabilities (2024)
\n
• Spider: Any-to-Many Multimodal LLM (2024)
\n
• Continuous Speech Tokens Makes LLMs Robust Multi-Modality Learners (2024)
\n

\n

Please give a thumbs up to this comment if you found it helpful!

\n

If you want recommendations for any Paper on Hugging Face checkout this Space

\n

You can directly ask Librarian Bot for paper recommendations by tagging it in a comment: \n\n@librarian-bot\n\t recommend

\n","updatedAt":"2024-12-14T01:35:39.316Z","author":{"_id":"63d3e0e8ff1384ce6c5dd17d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg","fullname":"Librarian Bot (Bot)","name":"librarian-bot","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":318,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.6740794777870178},"editors":["librarian-bot"],"editorAvatarUrls":["https://cdn-avatars.huggingface.co/v1/production/uploads/1674830754237-63d3e0e8ff1384ce6c5dd17d.jpeg"],"reactions":[],"isReport":false}},{"id":"675fc9ef88de8c1c1b3a2cbd","author":{"_id":"669cefd6119595d21b55a995","avatarUrl":"/avatars/bafc2387ee70b263bf45c42159381da8.svg","fullname":"Yuqi Liu","name":"Ricky06662","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":4,"isUserFollowing":false},"createdAt":"2024-12-16T06:34:23.000Z","type":"comment","data":{"edited":false,"hidden":false,"latest":{"raw":"Wow! awesome work.","html":"

Wow! awesome work.

\n","updatedAt":"2024-12-16T06:34:23.549Z","author":{"_id":"669cefd6119595d21b55a995","avatarUrl":"/avatars/bafc2387ee70b263bf45c42159381da8.svg","fullname":"Yuqi Liu","name":"Ricky06662","type":"user","isPro":false,"isHf":false,"isHfAdmin":false,"isMod":false,"followerCount":4,"isUserFollowing":false}},"numEdits":0,"identifiedLanguage":{"language":"en","probability":0.9343669414520264},"editors":["Ricky06662"],"editorAvatarUrls":["/avatars/bafc2387ee70b263bf45c42159381da8.svg"],"reactions":[],"isReport":false}}],"primaryEmailConfirmed":false,"paper":{"id":"2412.09501","authors":[{"_id":"675bcad776d0e555d631381c","user":{"_id":"65d882d30f35ed3f52d3ae2c","avatarUrl":"/avatars/22cda67c3fcd7150320ec3551eda90f5.svg","isPro":false,"fullname":"Zhisheng Zhong","user":"zszhong","type":"user"},"name":"Zhisheng Zhong","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:52:26.208Z","hidden":false},{"_id":"675bcad776d0e555d631381d","user":{"_id":"6423e35b30b0e4ab36dd1b16","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6423e35b30b0e4ab36dd1b16/pea6LVDS9PQAxvQt9GUiZ.jpeg","isPro":false,"fullname":"Wang Chengyao","user":"wcy1122","type":"user"},"name":"Chengyao Wang","status":"claimed_verified","statusLastChangedAt":"2024-12-13T15:21:46.575Z","hidden":false},{"_id":"675bcad776d0e555d631381e","name":"Yuqi Liu","hidden":false},{"_id":"675bcad776d0e555d631381f","user":{"_id":"6527b7280ae663e384eb8499","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6527b7280ae663e384eb8499/73yF3eu2cUx7jVZrhXnXx.jpeg","isPro":false,"fullname":"Senqiao Yang","user":"Senqiao","type":"user"},"name":"Senqiao Yang","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:51:23.068Z","hidden":false},{"_id":"675bcad776d0e555d6313820","user":{"_id":"6492a0d8d4ae24c933ace44d","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6492a0d8d4ae24c933ace44d/FXYIucGnWkMDu4gHWy1qw.jpeg","isPro":false,"fullname":"Longxiang Tang","user":"lloong","type":"user"},"name":"Longxiang Tang","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:50:10.607Z","hidden":false},{"_id":"675bcad776d0e555d6313821","user":{"_id":"6418554a0956be7233a1023e","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6418554a0956be7233a1023e/9EKN0GoOpcDbvBDmAQEJf.png","isPro":false,"fullname":"zhang yuechen","user":"julianjuaner","type":"user"},"name":"Yuechen Zhang","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:50:22.552Z","hidden":false},{"_id":"675bcad776d0e555d6313822","user":{"_id":"647ff779d037515be1169e35","avatarUrl":"/avatars/741096302cbdeb3670a8f31cf2d31d5d.svg","isPro":false,"fullname":"Jingyao Li","user":"JingyaoLi","type":"user"},"name":"Jingyao Li","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:50:51.188Z","hidden":false},{"_id":"675bcad776d0e555d6313823","user":{"_id":"66e79b3c1c79fc2e51dc1d60","avatarUrl":"/avatars/8706336e9e7a417505c9bb32583a662f.svg","isPro":false,"fullname":"QU Tianyuan","user":"TainU","type":"user"},"name":"Tianyuan Qu","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:45:31.921Z","hidden":false},{"_id":"675bcad776d0e555d6313824","user":{"_id":"643ff78cdc984afcbbbc3b1a","avatarUrl":"/avatars/eec5198ce88aaf8156840bec0d190a7f.svg","isPro":false,"fullname":"Yanwei Li","user":"YanweiLi","type":"user"},"name":"Yanwei Li","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:45:47.023Z","hidden":false},{"_id":"675bcad776d0e555d6313825","user":{"_id":"65b31d77f32c79fea79ba4d1","avatarUrl":"/avatars/8c51c1ff337ac372c7caafa4fa5019dc.svg","isPro":false,"fullname":"Yukang Chen","user":"Yukang2","type":"user"},"name":"Yukang Chen","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:50:59.539Z","hidden":false},{"_id":"675bcad776d0e555d6313826","name":"Shaozuo Yu","hidden":false},{"_id":"675bcad776d0e555d6313827","user":{"_id":"64119af4ca1bb1e9253b47f1","avatarUrl":"/avatars/aef1ce8822e66d342f541292f5e9fedf.svg","isPro":false,"fullname":"wusitong","user":"stonewst","type":"user"},"name":"Sitong Wu","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:51:15.558Z","hidden":false},{"_id":"675bcad776d0e555d6313828","user":{"_id":"6447485270338c0376087917","avatarUrl":"/avatars/359d17963a6d1c5f1fddf027169332a5.svg","isPro":false,"fullname":"Eric Lo","user":"ericlo","type":"user"},"name":"Eric Lo","status":"admin_assigned","statusLastChangedAt":"2024-12-16T10:49:59.436Z","hidden":false},{"_id":"675bcad776d0e555d6313829","name":"Shu Liu","hidden":false},{"_id":"675bcad776d0e555d631382a","name":"Jiaya Jia","hidden":false}],"mediaUrls":["https://cdn-uploads.huggingface.co/production/uploads/6423e35b30b0e4ab36dd1b16/QQC4kzQ5cQ9oPsd8m3R0P.mp4"],"publishedAt":"2024-12-12T17:50:39.000Z","submittedOnDailyAt":"2024-12-13T03:28:45.187Z","title":"Lyra: An Efficient and Speech-Centric Framework for Omni-Cognition","submittedOnDailyBy":{"_id":"6423e35b30b0e4ab36dd1b16","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6423e35b30b0e4ab36dd1b16/pea6LVDS9PQAxvQt9GUiZ.jpeg","isPro":false,"fullname":"Wang Chengyao","user":"wcy1122","type":"user"},"summary":"As Multi-modal Large Language Models (MLLMs) evolve, expanding beyond\nsingle-domain capabilities is essential to meet the demands for more versatile\nand efficient AI. However, previous omni-models have insufficiently explored\nspeech, neglecting its integration with multi-modality. We introduce Lyra, an\nefficient MLLM that enhances multimodal abilities, including advanced\nlong-speech comprehension, sound understanding, cross-modality efficiency, and\nseamless speech interaction. To achieve efficiency and speech-centric\ncapabilities, Lyra employs three strategies: (1) leveraging existing\nopen-source large models and a proposed multi-modality LoRA to reduce training\ncosts and data requirements; (2) using a latent multi-modality regularizer and\nextractor to strengthen the relationship between speech and other modalities,\nthereby enhancing model performance; and (3) constructing a high-quality,\nextensive dataset that includes 1.5M multi-modal (language, vision, audio) data\nsamples and 12K long speech samples, enabling Lyra to handle complex long\nspeech inputs and achieve more robust omni-cognition. Compared to other\nomni-methods, Lyra achieves state-of-the-art performance on various\nvision-language, vision-speech, and speech-language benchmarks, while also\nusing fewer computational resources and less training data.","upvotes":48,"discussionId":"675bcad876d0e555d6313861","githubRepo":"https://github.com/dvlab-research/Lyra","githubRepoAddedBy":"auto","ai_keywords":["Multi-modal Large Language Models (MLLMs)","omni-models","long-speech comprehension","sound understanding","cross-modality efficiency","seamless speech interaction","multi-modality LoRA","latent multi-modality regularizer","latent multi-modality extractor","omni-cognition","vision-language benchmarks","vision-speech benchmarks","speech-language benchmarks"],"githubStars":305},"canReadDatabase":false,"canManagePapers":false,"canSubmit":false,"hasHfLevelAccess":false,"upvoted":false,"upvoters":[{"_id":"6423e35b30b0e4ab36dd1b16","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6423e35b30b0e4ab36dd1b16/pea6LVDS9PQAxvQt9GUiZ.jpeg","isPro":false,"fullname":"Wang Chengyao","user":"wcy1122","type":"user"},{"_id":"644e1b1d9b4e87c31bab0a14","avatarUrl":"/avatars/88bb4c4a67dc8958069e9014f5e73a0b.svg","isPro":false,"fullname":"Michael Barry","user":"MichaelBarryUK","type":"user"},{"_id":"6418554a0956be7233a1023e","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6418554a0956be7233a1023e/9EKN0GoOpcDbvBDmAQEJf.png","isPro":false,"fullname":"zhang yuechen","user":"julianjuaner","type":"user"},{"_id":"620783f24e28382272337ba4","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/620783f24e28382272337ba4/zkUveQPNiDfYjgGhuFErj.jpeg","isPro":false,"fullname":"GuoLiangTang","user":"Tommy930","type":"user"},{"_id":"643b19f8a856622f978df30f","avatarUrl":"/avatars/c82779fdf94f80cdb5020504f83c818b.svg","isPro":false,"fullname":"Yatharth Sharma","user":"YaTharThShaRma999","type":"user"},{"_id":"675c3858ca32e91d535e2a95","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/675c3858ca32e91d535e2a95/l-sSKpDfx8qt9yziQj5J0.jpeg","isPro":false,"fullname":"Jiawen Chen","user":"ChenJiawen00","type":"user"},{"_id":"65d882d30f35ed3f52d3ae2c","avatarUrl":"/avatars/22cda67c3fcd7150320ec3551eda90f5.svg","isPro":false,"fullname":"Zhisheng Zhong","user":"zszhong","type":"user"},{"_id":"648eb1eb59c4e5c87dc116e0","avatarUrl":"/avatars/c636cea39c2c0937f01398c94ead5dad.svg","isPro":false,"fullname":"fdsqefsgergd","user":"T-representer","type":"user"},{"_id":"6270324ebecab9e2dcf245de","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/6270324ebecab9e2dcf245de/cMbtWSasyNlYc9hvsEEzt.jpeg","isPro":false,"fullname":"Kye Gomez","user":"kye","type":"user"},{"_id":"643ff78cdc984afcbbbc3b1a","avatarUrl":"/avatars/eec5198ce88aaf8156840bec0d190a7f.svg","isPro":false,"fullname":"Yanwei Li","user":"YanweiLi","type":"user"},{"_id":"652965773a416e1f2173443b","avatarUrl":"https://cdn-avatars.huggingface.co/v1/production/uploads/652965773a416e1f2173443b/y9MB8YgHzbwCXAc4EI9T3.jpeg","isPro":true,"fullname":"Yuhao Dong","user":"THUdyh","type":"user"},{"_id":"65decc75beffeb39ba679eba","avatarUrl":"/avatars/735b678bd5863a0c1b1bdd3bbf8858fa.svg","isPro":true,"fullname":"r","user":"oceansweep","type":"user"}],"acceptLanguages":["*"],"dailyPaperRank":0}">