{"id":"https://openalex.org/W4404652592","doi":"https://doi.org/10.48550/arxiv.2411.14062","title":"MMGenBench: Fully Automatically Evaluating LMMs from the Text-to-Image Generation Perspective","display_name":"MMGenBench: Fully Automatically Evaluating LMMs from the Text-to-Image Generation Perspective","publication_year":2024,"publication_date":"2024-11-21","ids":{"openalex":"https://openalex.org/W4404652592","doi":"https://doi.org/10.48550/arxiv.2411.14062"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2411.14062","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.14062","pdf_url":"https://arxiv.org/pdf/2411.14062","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2411.14062","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031578713","display_name":"Hailang Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Hailang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5115595919","display_name":"Yong Wang","orcid":"https://orcid.org/0009-0001-0305-2995"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Yong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101528257","display_name":"Zixuan Huang","orcid":"https://orcid.org/0000-0002-5028-4684"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Zixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041066318","display_name":"H. Li","orcid":"https://orcid.org/0009-0002-4036-4098"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Huaqiu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5014470040","display_name":"Tongwen Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Tongwen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101512474","display_name":"Xiangxiang Chu","orcid":"https://orcid.org/0000-0003-2548-0605"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chu, Xiangxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5027015677","display_name":"Richong Zhang","orcid":"https://orcid.org/0000-0002-1207-0300"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Richong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.9933000206947327,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13523","display_name":"Mathematics, Computing, and Information Processing","score":0.9172999858856201,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/perspective","display_name":"Perspective (graphical)","score":0.7987001538276672},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.5974785089492798},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44731009006500244},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.39624178409576416},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.3514458239078522},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3268001675605774},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2942165732383728}],"concepts":[{"id":"https://openalex.org/C12713177","wikidata":"https://www.wikidata.org/wiki/Q1900281","display_name":"Perspective (graphical)","level":2,"score":0.7987001538276672},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.5974785089492798},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44731009006500244},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.39624178409576416},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3514458239078522},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3268001675605774},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2942165732383728}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2411.14062","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.14062","pdf_url":"https://arxiv.org/pdf/2411.14062","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2411.14062","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2411.14062","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2411.14062","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2411.14062","pdf_url":"https://arxiv.org/pdf/2411.14062","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2899084033","https://openalex.org/W2018871932","https://openalex.org/W641279757","https://openalex.org/W370975646","https://openalex.org/W1670566515","https://openalex.org/W4242022592","https://openalex.org/W596972243","https://openalex.org/W2149537132","https://openalex.org/W2033914206","https://openalex.org/W2042327336"],"abstract_inverted_index":{"Large":[0],"Multimodal":[1],"Models":[2],"(LMMs)":[3],"demonstrate":[4],"impressive":[5],"capabilities.":[6],"However,":[7],"current":[8,165],"benchmarks":[9,20,141],"predominantly":[10],"focus":[11],"on":[12,109],"image":[13,104,111,151,188],"comprehension":[14],"in":[15,139,164],"specific":[16],"domains,":[17],"and":[18,57,81,86,106,124,130,153,167],"these":[19,49,71],"are":[21],"labor-intensive":[22],"to":[23,29,35,41,73,90,143,150],"construct.":[24],"Moreover,":[25],"their":[26],"answers":[27],"tend":[28],"be":[30],"brief,":[31],"making":[32],"it":[33],"difficult":[34],"assess":[36,178],"the":[37,53,84,92,122,128,146,158,179],"ability":[38],"of":[39,45,94,126,181],"LMMs":[40,100,120,137,166,182],"generate":[42],"detailed":[43],"descriptions":[44,66,72],"images.":[46,88],"To":[47],"address":[48],"limitations,":[50],"we":[51,96],"propose":[52],"MMGenBench-Pipeline,":[54,95],"a":[55],"straightforward":[56],"fully":[58],"automated":[59],"evaluation":[60,115],"pipeline.":[61],"This":[62,155],"involves":[63],"generating":[64],"textual":[65],"from":[67],"input":[68],"images,":[69],"using":[70,186],"create":[74],"auxiliary":[75],"images":[76],"via":[77],"text-to-image":[78],"generative":[79,110],"models,":[80],"then":[82],"comparing":[83],"original":[85],"generated":[87],"Furthermore,":[89],"ensure":[91],"effectiveness":[93,123],"design":[97],"MMGenBench-Test,":[98],"evaluating":[99],"across":[101,183],"13":[102],"distinct":[103],"patterns,":[105],"MMGenBench-Domain,":[107],"focusing":[108],"performance.":[112],"A":[113],"thorough":[114],"involving":[116],"over":[117],"50":[118],"popular":[119],"demonstrates":[121],"reliability":[125],"both":[127],"pipeline":[129],"benchmark.":[131],"Our":[132],"observations":[133],"indicate":[134],"that":[135],"numerous":[136],"excelling":[138],"existing":[140],"fail":[142],"adequately":[144],"complete":[145],"basic":[147],"tasks":[148],"related":[149],"understanding":[152],"description.":[154],"finding":[156],"highlights":[157],"substantial":[159],"potential":[160],"for":[161,170],"performance":[162,180],"improvement":[163],"suggests":[168],"avenues":[169],"future":[171],"model":[172],"optimization.":[173],"Concurrently,":[174],"MMGenBench-Pipeline":[175],"can":[176],"efficiently":[177],"diverse":[184],"domains":[185],"only":[187],"inputs.":[189]},"counts_by_year":[],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2024-11-24T00:00:00"}