{"id":"https://openalex.org/W4396681775","doi":"https://doi.org/10.1561/0600000110","title":"Multimodal Foundation Models: From Specialists to General-Purpose Assistants","display_name":"Multimodal Foundation Models: From Specialists to General-Purpose Assistants","publication_year":2024,"publication_date":"2024-05-06","ids":{"openalex":"https://openalex.org/W4396681775","doi":"https://doi.org/10.1561/0600000110"},"language":"en","primary_location":{"id":"doi:10.1561/0600000110","is_oa":true,"landing_page_url":"https://doi.org/10.1561/0600000110","pdf_url":null,"source":{"id":"https://openalex.org/S83210289","display_name":"Foundations and Trends\u00ae in Computer Graphics and Vision","issn_l":"1572-2740","issn":["1572-2740","1572-2759"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Computer Graphics and Vision","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://doi.org/10.1561/0600000110","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5107893340","display_name":"Chunyuan Li","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Chunyuan Li","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066666034","display_name":"Zhe Gan","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Zhe Gan","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050209478","display_name":"Zhengyuan Yang","orcid":"https://orcid.org/0000-0002-5808-0889"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Zhengyuan Yang","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114159502","display_name":"Jianwei Yang","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Jianwei Yang","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100657555","display_name":"Linjie Li","orcid":"https://orcid.org/0000-0003-0867-8863"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Linjie Li","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100780208","display_name":"Lijuan Wang","orcid":"https://orcid.org/0000-0002-6120-6200"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Lijuan Wang","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5114910293","display_name":"Jianfeng Gao","orcid":"https://orcid.org/0000-0002-5702-6143"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4210105678","display_name":"Microsoft (Finland)","ror":"https://ror.org/01nehjf29","country_code":"FI","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4210105678"]}],"countries":["FI","US"],"is_corresponding":false,"raw_author_name":"Jianfeng Gao","raw_affiliation_strings":["Microsoft Corporation ,","Microsoft Corporation, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Corporation ,","institution_ids":["https://openalex.org/I4210105678"]},{"raw_affiliation_string":"Microsoft Corporation, USA","institution_ids":["https://openalex.org/I1290206253"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":26.8068,"has_fulltext":false,"cited_by_count":109,"citation_normalized_percentile":{"value":0.99746895,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":99,"max":100},"biblio":{"volume":"16","issue":"1-2","first_page":"1","last_page":"214"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.3910999894142151,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10215","display_name":"Semantic Web and Ontologies","score":0.3910999894142151,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10456","display_name":"Multi-Agent Systems and Negotiation","score":0.35100001096725464,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/foundation","display_name":"Foundation (evidence)","score":0.7925788760185242},{"id":"https://openalex.org/keywords/medicine","display_name":"Medicine","score":0.37448036670684814},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3362254798412323},{"id":"https://openalex.org/keywords/geography","display_name":"Geography","score":0.15770328044891357},{"id":"https://openalex.org/keywords/archaeology","display_name":"Archaeology","score":0.08752909302711487}],"concepts":[{"id":"https://openalex.org/C2780966255","wikidata":"https://www.wikidata.org/wiki/Q5474306","display_name":"Foundation (evidence)","level":2,"score":0.7925788760185242},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.37448036670684814},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3362254798412323},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.15770328044891357},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.08752909302711487}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1561/0600000110","is_oa":true,"landing_page_url":"https://doi.org/10.1561/0600000110","pdf_url":null,"source":{"id":"https://openalex.org/S83210289","display_name":"Foundations and Trends\u00ae in Computer Graphics and Vision","issn_l":"1572-2740","issn":["1572-2740","1572-2759"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Computer Graphics and Vision","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1561/0600000110","is_oa":true,"landing_page_url":"https://doi.org/10.1561/0600000110","pdf_url":null,"source":{"id":"https://openalex.org/S83210289","display_name":"Foundations and Trends\u00ae in Computer Graphics and Vision","issn_l":"1572-2740","issn":["1572-2740","1572-2759"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310318575","host_organization_name":"Now Publishers","host_organization_lineage":["https://openalex.org/P4310318575"],"host_organization_lineage_names":["Now Publishers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Foundations and Trends\u00ae in Computer Graphics and Vision","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":166,"referenced_works":["https://openalex.org/W125693051","https://openalex.org/W1536680647","https://openalex.org/W1773149199","https://openalex.org/W1861492603","https://openalex.org/W1895577753","https://openalex.org/W1903029394","https://openalex.org/W1933349210","https://openalex.org/W1987983010","https://openalex.org/W1995903777","https://openalex.org/W2086052791","https://openalex.org/W2108598243","https://openalex.org/W2111993661","https://openalex.org/W2116719896","https://openalex.org/W2117539524","https://openalex.org/W2183182206","https://openalex.org/W2193145675","https://openalex.org/W2194775991","https://openalex.org/W2250384498","https://openalex.org/W2251512949","https://openalex.org/W2252136820","https://openalex.org/W2415053570","https://openalex.org/W2463955103","https://openalex.org/W2489434015","https://openalex.org/W2745461083","https://openalex.org/W2765811365","https://openalex.org/W2789366140","https://openalex.org/W2798991696","https://openalex.org/W2799263800","https://openalex.org/W2883725317","https://openalex.org/W2886641317","https://openalex.org/W2904565150","https://openalex.org/W2962784628","https://openalex.org/W2962843773","https://openalex.org/W2962898354","https://openalex.org/W2963109634","https://openalex.org/W2963115613","https://openalex.org/W2963150697","https://openalex.org/W2963622213","https://openalex.org/W2964061924","https://openalex.org/W2964284374","https://openalex.org/W2966683369","https://openalex.org/W2970231061","https://openalex.org/W2970986510","https://openalex.org/W2979579363","https://openalex.org/W2983943451","https://openalex.org/W2984008963","https://openalex.org/W2985076077","https://openalex.org/W2992838650","https://openalex.org/W2997998901","https://openalex.org/W2999219213","https://openalex.org/W3004349648","https://openalex.org/W3013529009","https://openalex.org/W3039368862","https://openalex.org/W3090449556","https://openalex.org/W3091588028","https://openalex.org/W3095670406","https://openalex.org/W3096609285","https://openalex.org/W3096831136","https://openalex.org/W3097217077","https://openalex.org/W3098267758","https://openalex.org/W3106546328","https://openalex.org/W3106859150","https://openalex.org/W3108655343","https://openalex.org/W3109684201","https://openalex.org/W3109931228","https://openalex.org/W3115656231","https://openalex.org/W3118212025","https://openalex.org/W3138516171","https://openalex.org/W3139418495","https://openalex.org/W3143320354","https://openalex.org/W3145450063","https://openalex.org/W3156892778","https://openalex.org/W3159481202","https://openalex.org/W3159619744","https://openalex.org/W3171007011","https://openalex.org/W3172845486","https://openalex.org/W3172942063","https://openalex.org/W3173220247","https://openalex.org/W3174770825","https://openalex.org/W3176641147","https://openalex.org/W3176824248","https://openalex.org/W3180355996","https://openalex.org/W3182683290","https://openalex.org/W3184784418","https://openalex.org/W3199693760","https://openalex.org/W3213454282","https://openalex.org/W4206994504","https://openalex.org/W4210352519","https://openalex.org/W4225683910","https://openalex.org/W4226321975","https://openalex.org/W4312230726","https://openalex.org/W4312309398","https://openalex.org/W4312420092","https://openalex.org/W4312563428","https://openalex.org/W4312691946","https://openalex.org/W4312708649","https://openalex.org/W4312777209","https://openalex.org/W4312784228","https://openalex.org/W4312804044","https://openalex.org/W4312846625","https://openalex.org/W4312891522","https://openalex.org/W4312910992","https://openalex.org/W4312911498","https://openalex.org/W4312920106","https://openalex.org/W4312933868","https://openalex.org/W4312935996","https://openalex.org/W4312956471","https://openalex.org/W4312960937","https://openalex.org/W4312980231","https://openalex.org/W4312992236","https://openalex.org/W4313026212","https://openalex.org/W4313045695","https://openalex.org/W4313131769","https://openalex.org/W4313136445","https://openalex.org/W4313156423","https://openalex.org/W4313158203","https://openalex.org/W4365483249","https://openalex.org/W4382465386","https://openalex.org/W4382467347","https://openalex.org/W4382567565","https://openalex.org/W4385270985","https://openalex.org/W4385570217","https://openalex.org/W4385573236","https://openalex.org/W4386057720","https://openalex.org/W4386057769","https://openalex.org/W4386065512","https://openalex.org/W4386065691","https://openalex.org/W4386066126","https://openalex.org/W4386071707","https://openalex.org/W4386072059","https://openalex.org/W4386075493","https://openalex.org/W4386076084","https://openalex.org/W4386076160","https://openalex.org/W4386076385","https://openalex.org/W4386076418","https://openalex.org/W4386076522","https://openalex.org/W4389519587","https://openalex.org/W4389519620","https://openalex.org/W4389520009","https://openalex.org/W4389520529","https://openalex.org/W4389523832","https://openalex.org/W4389524238","https://openalex.org/W4389539271","https://openalex.org/W4390872103","https://openalex.org/W4390872747","https://openalex.org/W4390873054","https://openalex.org/W4390873312","https://openalex.org/W4390873331","https://openalex.org/W4390873343","https://openalex.org/W4390873622","https://openalex.org/W4390873750","https://openalex.org/W4390873875","https://openalex.org/W4390874393","https://openalex.org/W4390874575","https://openalex.org/W4402753769","https://openalex.org/W6912494966","https://openalex.org/W6912895990","https://openalex.org/W7133182869","https://openalex.org/W7133196460","https://openalex.org/W7133208656","https://openalex.org/W7133220561","https://openalex.org/W7133224749","https://openalex.org/W7133233472","https://openalex.org/W7133240343","https://openalex.org/W7133241217","https://openalex.org/W7133242526"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W3031052312","https://openalex.org/W4389568370","https://openalex.org/W3032375762","https://openalex.org/W1995515455","https://openalex.org/W2080531066","https://openalex.org/W3108674512","https://openalex.org/W1506200166","https://openalex.org/W1489783725"],"abstract_inverted_index":{"This":[0],"monograph":[1,126],"presents":[2],"a":[3,46],"comprehensive":[4],"survey":[5,47],"of":[6,11,48,64,94,112,124],"the":[7,23,92,125,145],"taxonomy":[8],"and":[9,18,71,115,131,136,147],"evolution":[10],"multimodal":[12,52,85,113,117,138,151],"foundation":[13,53,86,152],"models":[14,27,54,87,103,108],"that":[15,88],"demonstrate":[16],"vision":[17,66,102,135],"vision-language":[19,137],"capabilities,":[20],"focusing":[21],"on":[22],"transition":[24],"from":[25],"specialist":[26],"to":[28,90,143],"generalpurpose":[29],"assistants.":[30],"The":[31,121],"research":[32,50,83],"landscape":[33],"encompasses":[34],"five":[35],"core":[36],"topics,":[37],"categorized":[38],"into":[39],"two":[40,60],"classes.":[41],"(i)":[42],"We":[43],"start":[44],"with":[45,119],"well-established":[49],"areas:":[51,84],"pre-trained":[55],"for":[56,68],"specific":[57],"purposes,":[58],"including":[59,97],"topics":[61,99],"\u2013":[62,100],"methods":[63],"learning":[65],"backbones":[67],"visual":[69],"understanding":[70],"text-to-image":[72],"generation.":[73],"(ii)":[74],"Then,":[75],"we":[76],"present":[77],"recent":[78,148],"advances":[79,149],"in":[80,133,150],"exploratory,":[81],"open":[82],"aim":[89],"play":[91],"role":[93],"general-purpose":[95],"assistants,":[96],"three":[98],"unified":[101],"inspired":[104],"by":[105],"large":[106],"language":[107],"(LLMs),":[109],"end-to-end":[110],"training":[111],"LLMs,":[114],"chaining":[116],"tools":[118],"LLMs.":[120],"target":[122],"audiences":[123],"are":[127,141],"researchers,":[128],"graduate":[129],"students,":[130],"professionals":[132],"computer":[134],"communities":[139],"who":[140],"eager":[142],"learn":[144],"basics":[146],"models.":[153]},"counts_by_year":[{"year":2026,"cited_by_count":23},{"year":2025,"cited_by_count":62},{"year":2024,"cited_by_count":24}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}