{"id":"https://openalex.org/W4415682262","doi":"https://doi.org/10.1007/s00138-025-01762-0","title":"Benchmarking large and small MLLMs","display_name":"Benchmarking large and small MLLMs","publication_year":2025,"publication_date":"2025-10-30","ids":{"openalex":"https://openalex.org/W4415682262","doi":"https://doi.org/10.1007/s00138-025-01762-0"},"language":"en","primary_location":{"id":"doi:10.1007/s00138-025-01762-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s00138-025-01762-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s00138-025-01762-0.pdf","source":{"id":"https://openalex.org/S27728525","display_name":"Machine Vision and Applications","issn_l":"0932-8092","issn":["0932-8092","1432-1769"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Vision and Applications","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s00138-025-01762-0.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Xuelu Feng","orcid":null},"institutions":[{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Xuelu Feng","raw_affiliation_strings":["Department of Computer Science and Engineering, University at Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University at Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I63190737"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079735821","display_name":"Y. Li","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yunsheng Li","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5040923451","display_name":"Dongdong Chen","orcid":"https://orcid.org/0009-0008-3966-3270"},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dongdong Chen","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115829672","display_name":"Mei Gao","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mei Gao","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Mengchen Liu","orcid":null},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Mengchen Liu","raw_affiliation_strings":["Microsoft, Redmond, WA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft, Redmond, WA, USA","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085245110","display_name":"Junsong Yuan","orcid":"https://orcid.org/0000-0002-7324-7034"},"institutions":[{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Junsong Yuan","raw_affiliation_strings":["Department of Computer Science and Engineering, University at Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University at Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I63190737"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5115829673","display_name":"Chunming Qiao","orcid":null},"institutions":[{"id":"https://openalex.org/I63190737","display_name":"University at Buffalo, State University of New York","ror":"https://ror.org/01y64my43","country_code":"US","type":"education","lineage":["https://openalex.org/I63190737"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chunming Qiao","raw_affiliation_strings":["Department of Computer Science and Engineering, University at Buffalo, Buffalo, NY, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science and Engineering, University at Buffalo, Buffalo, NY, USA","institution_ids":["https://openalex.org/I63190737"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I63190737"],"apc_list":{"value":2490,"currency":"EUR","value_usd":3090},"apc_paid":{"value":2490,"currency":"EUR","value_usd":3090},"fwci":2.2966,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":{"value":0.89931072,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":98},"biblio":{"volume":"36","issue":"6","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2833000123500824,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.2833000123500824,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.22360000014305115,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13702","display_name":"Machine Learning in Healthcare","score":0.03530000150203705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/benchmarking","display_name":"Benchmarking","score":0.8147000074386597},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.6190000176429749},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5789999961853027},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.5663999915122986},{"id":"https://openalex.org/keywords/usability","display_name":"Usability","score":0.5360999703407288},{"id":"https://openalex.org/keywords/best-practice","display_name":"Best practice","score":0.35670000314712524}],"concepts":[{"id":"https://openalex.org/C86251818","wikidata":"https://www.wikidata.org/wiki/Q816754","display_name":"Benchmarking","level":2,"score":0.8147000074386597},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.744700014591217},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.6190000176429749},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5789999961853027},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.5663999915122986},{"id":"https://openalex.org/C170130773","wikidata":"https://www.wikidata.org/wiki/Q216378","display_name":"Usability","level":2,"score":0.5360999703407288},{"id":"https://openalex.org/C2522767166","wikidata":"https://www.wikidata.org/wiki/Q2374463","display_name":"Data science","level":1,"score":0.47440001368522644},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.40939998626708984},{"id":"https://openalex.org/C184356942","wikidata":"https://www.wikidata.org/wiki/Q830382","display_name":"Best practice","level":2,"score":0.35670000314712524},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.34220001101493835},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.3294000029563904},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.31130000948905945},{"id":"https://openalex.org/C91262260","wikidata":"https://www.wikidata.org/wiki/Q528074","display_name":"End user","level":2,"score":0.2937000095844269},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.2854999899864197},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.28110000491142273},{"id":"https://openalex.org/C195094911","wikidata":"https://www.wikidata.org/wiki/Q14167904","display_name":"Process management","level":1,"score":0.271699994802475},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2676999866962433},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.2651999890804291}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s00138-025-01762-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s00138-025-01762-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s00138-025-01762-0.pdf","source":{"id":"https://openalex.org/S27728525","display_name":"Machine Vision and Applications","issn_l":"0932-8092","issn":["0932-8092","1432-1769"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Vision and Applications","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s00138-025-01762-0","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s00138-025-01762-0","pdf_url":"https://link.springer.com/content/pdf/10.1007/s00138-025-01762-0.pdf","source":{"id":"https://openalex.org/S27728525","display_name":"Machine Vision and Applications","issn_l":"0932-8092","issn":["0932-8092","1432-1769"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Machine Vision and Applications","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G2464345056","display_name":null,"funder_award_id":"2229873","funder_id":"https://openalex.org/F4320332210","funder_display_name":"Institute of Education Sciences"},{"id":"https://openalex.org/G3999138594","display_name":null,"funder_award_id":"CNS-2120369","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G473008951","display_name":"Collaborative Research: CCRI: New: Medium: A Development and Experimental Environment for Privacy-preserving and Secure (DEEPSECURE) Machine Learning","funder_award_id":"2120369","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5642188625","display_name":"AI Institute for Transforming Education for Children with Speech and Language Processing Challenges","funder_award_id":"2229873","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332210","display_name":"Institute of Education Sciences","ror":"https://ror.org/04et59085"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415682262.pdf","grobid_xml":"https://content.openalex.org/works/W4415682262.grobid-xml"},"referenced_works_count":34,"referenced_works":["https://openalex.org/W12634471","https://openalex.org/W162608378","https://openalex.org/W1861492603","https://openalex.org/W2136451880","https://openalex.org/W2163446794","https://openalex.org/W2263785794","https://openalex.org/W2295107390","https://openalex.org/W2340897893","https://openalex.org/W2515770085","https://openalex.org/W2531044305","https://openalex.org/W2561715562","https://openalex.org/W2948982773","https://openalex.org/W2962832028","https://openalex.org/W2963150697","https://openalex.org/W2963690694","https://openalex.org/W2963758027","https://openalex.org/W2988326850","https://openalex.org/W3170764266","https://openalex.org/W3196731672","https://openalex.org/W3202415077","https://openalex.org/W4295006274","https://openalex.org/W4312230726","https://openalex.org/W4319456044","https://openalex.org/W4319996536","https://openalex.org/W4378083716","https://openalex.org/W4390874575","https://openalex.org/W4400125901","https://openalex.org/W4402716381","https://openalex.org/W4402726948","https://openalex.org/W4403081466","https://openalex.org/W4403923795","https://openalex.org/W4404575065","https://openalex.org/W4404781890","https://openalex.org/W4405399726"],"related_works":[],"abstract_inverted_index":{"Abstract":[0],"Large":[1],"multimodal":[2,19,117],"language":[3],"models":[4,57,142,175],"(MLLMs)":[5],"such":[6,110],"as":[7,111,119,121],"GPT-4V":[8],"and":[9,17,24,41,58,69,85,97,104,116,128,167,196],"GPT-4o":[10],"have":[11],"achieved":[12],"remarkable":[13],"advancements":[14],"in":[15,124,143,149,164,186],"understanding":[16],"generating":[18],"content,":[20],"showcasing":[21],"superior":[22],"quality":[23,189],"capabilities":[25,109],"across":[26,198],"diverse":[27,199],"tasks.":[28],"However,":[29],"their":[30,77,194],"deployment":[31,67],"faces":[32],"significant":[33],"challenges,":[34],"including":[35],"slow":[36],"inference,":[37,65],"high":[38],"computational":[39],"cost,":[40],"impracticality":[42],"for":[43],"on-device":[44],"applications.":[45,200],"In":[46,90],"contrast,":[47],"the":[48,55,70,80,183,188],"emergence":[49],"of":[50,191],"small":[51,86,103,134,166],"MLLMs,":[52,106,169,192],"exemplified":[53],"by":[54],"LLava-series":[56],"Phi-3-Vision,":[59],"offers":[60],"promising":[61],"alternatives":[62],"with":[63],"faster":[64],"reduced":[66],"costs,":[68],"ability":[71],"to":[72,100,140],"handle":[73],"domain-specific":[74],"scenarios.":[75],"Despite":[76],"growing":[78],"presence,":[79],"capability":[81],"boundaries":[82,190],"between":[83],"large":[84,105,141,168],"MLLMs":[87,135],"remain":[88],"underexplored.":[89],"this":[91],"work,":[92],"we":[93,159],"conduct":[94],"a":[95],"systematic":[96],"comprehensive":[98],"evaluation":[99,131],"benchmark":[101],"both":[102,165],"spanning":[107],"general":[108],"object":[112],"recognition,":[113],"temporal":[114],"reasoning,":[115],"comprehension,":[118],"well":[120],"real-world":[122],"applications":[123],"domains":[125,171],"like":[126],"industry":[127],"automotive.":[129],"Our":[130],"reveals":[132],"that":[133],"can":[136],"achieve":[137],"comparable":[138],"performance":[139],"specific":[144],"scenarios":[145],"but":[146],"lag":[147],"significantly":[148],"complex":[150],"tasks":[151],"requiring":[152],"deeper":[153],"reasoning":[154],"or":[155],"nuanced":[156],"understanding.":[157],"Furthermore,":[158],"identify":[160],"common":[161],"failure":[162],"cases":[163],"highlighting":[170],"where":[172],"even":[173],"state-of-the-art":[174],"struggle.":[176],"We":[177],"hope":[178],"our":[179],"findings":[180],"will":[181],"guide":[182],"research":[184],"community":[185],"pushing":[187],"advancing":[193],"usability":[195],"effectiveness":[197]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-13T06:13:01.061226","created_date":"2025-10-30T00:00:00"}