{"id":"https://openalex.org/W4386655647","doi":"https://doi.org/10.48550/arxiv.2309.05519","title":"NExT-GPT: Any-to-Any Multimodal LLM","display_name":"NExT-GPT: Any-to-Any Multimodal LLM","publication_year":2023,"publication_date":"2023-09-11","ids":{"openalex":"https://openalex.org/W4386655647","doi":"https://doi.org/10.48550/arxiv.2309.05519"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2309.05519","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.05519","pdf_url":"https://arxiv.org/pdf/2309.05519","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2309.05519","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089517967","display_name":"Shengqiong Wu","orcid":"https://orcid.org/0000-0001-6192-1194"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wu, Shengqiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Fei, Hao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fei, Hao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102773468","display_name":"Leigang Qu","orcid":"https://orcid.org/0000-0002-5479-8439"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qu, Leigang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075066245","display_name":"Wei Ji","orcid":"https://orcid.org/0000-0002-9642-6983"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ji, Wei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5089404640","display_name":"Tat\u2010Seng Chua","orcid":"https://orcid.org/0000-0001-6097-7807"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chua, Tat-Seng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":95,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9984999895095825,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9962000250816345,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9912999868392944,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/modalities","display_name":"Modalities","score":0.7986636161804199},{"id":"https://openalex.org/keywords/modality","display_name":"Modality (human\u2013computer interaction)","score":0.753498911857605},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7275862097740173},{"id":"https://openalex.org/keywords/encoder","display_name":"Encoder","score":0.5256905555725098},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.49487370252609253},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.408376544713974},{"id":"https://openalex.org/keywords/multimedia","display_name":"Multimedia","score":0.3683393597602844}],"concepts":[{"id":"https://openalex.org/C2779903281","wikidata":"https://www.wikidata.org/wiki/Q6888026","display_name":"Modalities","level":2,"score":0.7986636161804199},{"id":"https://openalex.org/C2780226545","wikidata":"https://www.wikidata.org/wiki/Q6888030","display_name":"Modality (human\u2013computer interaction)","level":2,"score":0.753498911857605},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7275862097740173},{"id":"https://openalex.org/C118505674","wikidata":"https://www.wikidata.org/wiki/Q42586063","display_name":"Encoder","level":2,"score":0.5256905555725098},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.49487370252609253},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.408376544713974},{"id":"https://openalex.org/C49774154","wikidata":"https://www.wikidata.org/wiki/Q131765","display_name":"Multimedia","level":1,"score":0.3683393597602844},{"id":"https://openalex.org/C36289849","wikidata":"https://www.wikidata.org/wiki/Q34749","display_name":"Social science","level":1,"score":0.0},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2309.05519","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.05519","pdf_url":"https://arxiv.org/pdf/2309.05519","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},{"id":"doi:10.48550/arxiv.2309.05519","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2309.05519","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2309.05519","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2309.05519","pdf_url":"https://arxiv.org/pdf/2309.05519","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320318547","display_name":"Baidu","ror":"https://ror.org/03vs3wt56"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4386655647.pdf","grobid_xml":"https://content.openalex.org/works/W4386655647.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W73545470","https://openalex.org/W4224266612","https://openalex.org/W2383394264","https://openalex.org/W4320153225","https://openalex.org/W4293261942","https://openalex.org/W3125968744","https://openalex.org/W2167701463","https://openalex.org/W2110287964","https://openalex.org/W4307407935","https://openalex.org/W649759291"],"abstract_inverted_index":{"While":[0],"recently":[1],"Multimodal":[2],"Large":[3],"Language":[4],"Models":[5],"(MM-LLMs)":[6],"have":[7],"made":[8],"exciting":[9],"strides,":[10],"they":[11],"mostly":[12],"fall":[13],"prey":[14],"to":[15,26,60,89,139],"the":[16,24,37,65,106,177,191,199],"limitation":[17],"of":[18,50,98,121,124,180,186],"only":[19,117,130],"input-side":[20],"multimodal":[21,81],"understanding,":[22],"without":[23],"ability":[25],"produce":[27],"content":[28,54,171],"in":[29,55,95,198],"multiple":[30],"modalities.":[31,142],"As":[32],"we":[33,67,144],"humans":[34],"always":[35],"perceive":[36,90],"world":[38],"and":[39,52,83,92,102,111,134,151,170],"communicate":[40],"with":[41,80,116,165],"people":[42],"through":[43],"various":[44],"modalities,":[45,189],"developing":[46],"any-to-any":[47,72],"MM-LLMs":[48],"capable":[49,185],"accepting":[51],"delivering":[53],"any":[56],"modality":[57],"becomes":[58],"essential":[59],"human-level":[61],"AI.":[62],"To":[63],"fill":[64],"gap,":[66],"present":[68],"an":[69,78,182],"end-to-end":[70],"general-purpose":[71],"MM-LLM":[73],"system,":[74],"NExT-GPT.":[75],"We":[76],"connect":[77],"LLM":[79],"adaptors":[82],"different":[84],"diffusion":[85],"decoders,":[86,112],"enabling":[87],"NExT-GPT":[88,113,162],"inputs":[91],"generate":[93],"outputs":[94],"arbitrary":[96],"combinations":[97],"text,":[99],"images,":[100],"videos,":[101],"audio.":[103],"By":[104],"leveraging":[105],"existing":[107],"well-trained":[108],"highly-performing":[109],"encoders":[110],"is":[114,163],"tuned":[115],"a":[118,146,154],"small":[119],"amount":[120],"parameter":[122],"(1%)":[123],"certain":[125],"projection":[126],"layers,":[127],"which":[128,161],"not":[129],"benefits":[131],"low-cost":[132],"training":[133],"also":[135],"facilitates":[136],"convenient":[137],"expansion":[138],"more":[140,194],"potential":[141],"Moreover,":[143],"introduce":[145],"modality-switching":[147],"instruction":[148],"tuning":[149],"(MosIT)":[150],"manually":[152],"curate":[153],"high-quality":[155],"dataset":[156],"for":[157,193],"MosIT,":[158],"based":[159],"on":[160],"empowered":[164],"complex":[166],"cross-modal":[167],"semantic":[168],"understanding":[169],"generation.":[172],"Overall,":[173],"our":[174],"research":[175,197],"showcases":[176],"promising":[178],"possibility":[179],"building":[181],"AI":[183,196],"agent":[184],"modeling":[187],"universal":[188],"paving":[190],"way":[192],"human-like":[195],"community.":[200],"Project":[201],"page:":[202],"https://next-gpt.github.io/":[203]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":18},{"year":2024,"cited_by_count":62},{"year":2023,"cited_by_count":13},{"year":2021,"cited_by_count":1}],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2025-10-10T00:00:00"}