{"id":"https://openalex.org/W4406983024","doi":"https://doi.org/10.1109/tbdata.2025.3536930","title":"TinyLVLM-eHub: Towards Comprehensive and Efficient Evaluation for Large Vision-Language Models","display_name":"TinyLVLM-eHub: Towards Comprehensive and Efficient Evaluation for Large Vision-Language Models","publication_year":2025,"publication_date":"2025-01-30","ids":{"openalex":"https://openalex.org/W4406983024","doi":"https://doi.org/10.1109/tbdata.2025.3536930"},"language":"en","primary_location":{"id":"doi:10.1109/tbdata.2025.3536930","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tbdata.2025.3536930","pdf_url":null,"source":{"id":"https://openalex.org/S2491400915","display_name":"IEEE Transactions on Big Data","issn_l":"2332-7790","issn":["2332-7790","2372-2096"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Big Data","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101827257","display_name":"Wenqi Shao","orcid":"https://orcid.org/0000-0003-3781-4086"},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenqi Shao","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":"https://orcid.org/0000-0003-3781-4086","affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5112272361","display_name":"Meng Lei","orcid":null},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Meng Lei","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5015205154","display_name":"Yu\u2010Tao Hu","orcid":null},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yutao Hu","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Peng Gao","orcid":"https://orcid.org/0009-0005-7881-712X"},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Gao","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":"https://orcid.org/0009-0005-7881-712X","affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5110556683","display_name":"Peng Xu","orcid":"https://orcid.org/0009-0008-8742-2420"},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Peng Xu","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113257068","display_name":"Kaipeng Zhang","orcid":null},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kaipeng Zhang","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":"https://orcid.org/0000-0001-6105-6532","affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5104225369","display_name":"Fanqing Meng","orcid":null},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Fanqing Meng","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":"https://orcid.org/0000-0002-0920-3539","affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067080265","display_name":"Siyuan Huang","orcid":"https://orcid.org/0000-0003-1524-7148"},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Siyuan Huang","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100732450","display_name":"Hongsheng Li","orcid":"https://orcid.org/0000-0002-2664-7975"},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hongsheng Li","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":"https://orcid.org/0000-0002-2664-7975","affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100748135","display_name":"Yu Qiao","orcid":"https://orcid.org/0000-0002-1889-2567"},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu Qiao","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China","OpenGVLab, Shanghai AI Laboratory, China"],"raw_orcid":"https://orcid.org/0000-0002-1889-2567","affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]},{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]},{"author_position":"last","author":{"id":null,"display_name":"Ping Luo","orcid":null},"institutions":[{"id":"https://openalex.org/I4210111163","display_name":"Shanghai Open University","ror":"https://ror.org/027r7gj11","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210111163"]},{"id":"https://openalex.org/I4391012619","display_name":"Shanghai Artificial Intelligence Laboratory","ror":"https://ror.org/03wkvpx79","country_code":null,"type":"facility","lineage":["https://openalex.org/I4391012619"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ping Luo","raw_affiliation_strings":["OpenGVLab, Shanghai AI Laboratory, Shanghai, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"OpenGVLab, Shanghai AI Laboratory, Shanghai, China","institution_ids":["https://openalex.org/I4210111163","https://openalex.org/I4391012619"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":11,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.9349,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.71360167,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"11","issue":"3","first_page":"933","last_page":"947"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.9925000071525574,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9521999955177307,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.9272000193595886,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.832796573638916},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.39512529969215393}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.832796573638916},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39512529969215393}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tbdata.2025.3536930","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tbdata.2025.3536930","pdf_url":null,"source":{"id":"https://openalex.org/S2491400915","display_name":"IEEE Transactions on Big Data","issn_l":"2332-7790","issn":["2332-7790","2372-2096"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320439","host_organization_name":"IEEE Computer Society","host_organization_lineage":["https://openalex.org/P4310320439","https://openalex.org/P4310319808"],"host_organization_lineage_names":["IEEE Computer Society","Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Big Data","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W1971822075","https://openalex.org/W1977295328","https://openalex.org/W2000324040","https://openalex.org/W2008806374","https://openalex.org/W2049951199","https://openalex.org/W2090048052","https://openalex.org/W2117539524","https://openalex.org/W2144554289","https://openalex.org/W2146835493","https://openalex.org/W2533598788","https://openalex.org/W2799002257","https://openalex.org/W2914492226","https://openalex.org/W2947312908","https://openalex.org/W2963115613","https://openalex.org/W2963299604","https://openalex.org/W2963518342","https://openalex.org/W2979382951","https://openalex.org/W2986619406","https://openalex.org/W3003484198","https://openalex.org/W3004082545","https://openalex.org/W3004268082","https://openalex.org/W3120043490","https://openalex.org/W3202415716","https://openalex.org/W4313045695","https://openalex.org/W4381802186","https://openalex.org/W4386076522","https://openalex.org/W4389519200","https://openalex.org/W4389523832","https://openalex.org/W4390874161","https://openalex.org/W4403081466","https://openalex.org/W4404788931","https://openalex.org/W4405399726","https://openalex.org/W6639432524","https://openalex.org/W6691603626","https://openalex.org/W6787972765","https://openalex.org/W6803645040","https://openalex.org/W6810334672","https://openalex.org/W6811340617","https://openalex.org/W6843405348","https://openalex.org/W6846931871","https://openalex.org/W6849177959","https://openalex.org/W6850625674","https://openalex.org/W6851592950","https://openalex.org/W6851813333","https://openalex.org/W6851950068","https://openalex.org/W6852060543","https://openalex.org/W6852447913","https://openalex.org/W6852489829","https://openalex.org/W6852874933","https://openalex.org/W6853116092","https://openalex.org/W6853122427","https://openalex.org/W6853264062","https://openalex.org/W6853469104","https://openalex.org/W6853875005","https://openalex.org/W6854336665","https://openalex.org/W6854347851","https://openalex.org/W6854510102","https://openalex.org/W6854691855"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2899084033","https://openalex.org/W2748952813","https://openalex.org/W2390279801","https://openalex.org/W4391913857","https://openalex.org/W2358668433","https://openalex.org/W4396701345","https://openalex.org/W2376932109","https://openalex.org/W2001405890","https://openalex.org/W4396696052"],"abstract_inverted_index":{"Large":[0],"Vision-Language":[1],"Models":[2],"(LVLMs)":[3],"have":[4],"made":[5],"significant":[6],"strides":[7],"in":[8,111,124,134],"various":[9],"multimodal":[10,20,50,145],"tasks.":[11],"Notably,":[12],"GPT4V,":[13],"Claude,":[14],"Gemini,":[15],"and":[16,26,34,63,75,108,137],"others":[17],"showcase":[18],"exceptional":[19],"capabilities,":[21,51],"marked":[22],"by":[23],"profound":[24],"comprehension":[25],"reasoning":[27],"skills.":[28],"This":[29,127],"study":[30],"introduces":[31],"a":[32,73,140],"comprehensive":[33],"efficient":[35],"evaluation":[36,82,95,128],"framework,":[37],"TinyLVLM-eHub,":[38],"to":[39,97,115],"assess":[40],"LVLMs\u2019":[41],"performance,":[42],"including":[43],"proprietary":[44],"models.":[45],"TinyLVLM-eHub":[46],"covers":[47],"six":[48],"key":[49],"such":[52],"as":[53,139],"visual":[54],"perception,":[55],"knowledge":[56],"acquisition,":[57],"reasoning,":[58],"commonsense":[59],"understanding,":[60],"object":[61,125],"hallucination,":[62],"embodied":[64],"intelligence.":[65],"The":[66,81],"benchmark,":[67],"utilizing":[68],"2.1K":[69],"image-text":[70],"pairs,":[71],"provides":[72],"user-friendly":[74],"accessible":[76],"platform":[77],"for":[78,131,143],"LVLM":[79,132],"evaluation.":[80],"employs":[83],"the":[84],"ChatGPT":[85],"Ensemble":[86],"Evaluation":[87],"(CEE)":[88],"method,":[89],"which":[90],"improves":[91],"alignment":[92],"with":[93],"human":[94],"compared":[96,114],"word-matching":[98],"approaches.":[99],"Results":[100],"reveal":[101],"that":[102],"closed-source":[103],"API":[104],"models":[105],"like":[106],"GPT4V":[107],"GeminiPro-V":[109],"excel":[110],"most":[112],"capabilities":[113],"previous":[116],"open-source":[117],"LVLMs,":[118],"though":[119],"they":[120],"show":[121],"some":[122],"vulnerability":[123],"hallucination.":[126],"underscores":[129],"areas":[130],"improvement":[133],"real-world":[135],"applications":[136],"serves":[138],"foundational":[141],"assessment":[142],"future":[144],"advancements.":[146]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}