{"id":"https://openalex.org/W4402684064","doi":"https://doi.org/10.18653/v1/2024.findings-acl.962","title":"PuzzleVQA: Diagnosing Multimodal Reasoning Challenges of Language Models with Abstract Visual Patterns","display_name":"PuzzleVQA: Diagnosing Multimodal Reasoning Challenges of Language Models with Abstract Visual Patterns","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4402684064","doi":"https://doi.org/10.18653/v1/2024.findings-acl.962"},"language":"en","primary_location":{"id":"doi:10.18653/v1/2024.findings-acl.962","doi":"https://doi.org/10.18653/v1/2024.findings-acl.962","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.962","pdf_url":"https://aclanthology.org/2024.findings-acl.962.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://aclanthology.org/2024.findings-acl.962.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5087486426","display_name":"Yew Ken Chia","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I152815399","display_name":"Singapore University of Technology and Design","ror":"https://ror.org/05j6fvn87","country_code":"SG","type":"education","lineage":["https://openalex.org/I152815399"]},{"id":"https://openalex.org/I4405281272","display_name":"Alibaba DAMO Academy (China)","ror":"https://ror.org/0448px924","country_code":"CN","type":"company","lineage":["https://openalex.org/I4405281272","https://openalex.org/I45928872"]},{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN","SG"],"is_corresponding":false,"raw_author_name":"Yew Ken Chia","raw_affiliation_strings":["DAMO Academy , Alibaba Group , Singapore","Singapore University of Technology and Design ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"DAMO Academy , Alibaba Group , Singapore","institution_ids":["https://openalex.org/I4405281272","https://openalex.org/I45928872"]},{"raw_affiliation_string":"Singapore University of Technology and Design ,","institution_ids":["https://openalex.org/I152815399"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114099831","display_name":"Vernon Toh","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vernon Toh","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086877005","display_name":"Deepanway Ghosal","orcid":"https://orcid.org/0000-0002-3858-4449","observed_orcids":["https://orcid.org/0000-0002-3858-4449"]},"institutions":[{"id":"https://openalex.org/I152815399","display_name":"Singapore University of Technology and Design","ror":"https://ror.org/05j6fvn87","country_code":"SG","type":"education","lineage":["https://openalex.org/I152815399"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Deepanway Ghosal","raw_affiliation_strings":["Singapore University of Technology and Design ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore University of Technology and Design ,","institution_ids":["https://openalex.org/I152815399"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086674741","display_name":"Lidong Bing","orcid":"https://orcid.org/0000-0003-4565-6313","observed_orcids":["https://orcid.org/0000-0003-4565-6313"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lidong Bing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5033376109","display_name":"Soujanya Poria","orcid":"https://orcid.org/0000-0001-6924-7931","observed_orcids":["https://orcid.org/0000-0001-6924-7931"]},"institutions":[{"id":"https://openalex.org/I152815399","display_name":"Singapore University of Technology and Design","ror":"https://ror.org/05j6fvn87","country_code":"SG","type":"education","lineage":["https://openalex.org/I152815399"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Soujanya Poria","raw_affiliation_strings":["Singapore University of Technology and Design ,"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Singapore University of Technology and Design ,","institution_ids":["https://openalex.org/I152815399"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.2398,"has_fulltext":true,"cited_by_count":50,"citation_normalized_percentile":{"value":0.97857395,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"16259","last_page":"16273"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-multimodal-models","display_name":"large multimodal models","score":0.703000009059906},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"pattern recognition","score":0.5690000057220459},{"id":"https://openalex.org/keywords/g-factor","display_name":"general intelligence","score":0.4970000088214874},{"id":"https://openalex.org/keywords/visual-perception","display_name":"visual perception","score":0.6769999861717224},{"id":"https://openalex.org/keywords/inductive-reasoning","display_name":"inductive reasoning","score":0.9980000257492065},{"id":"https://openalex.org/keywords/deductive-reasoning","display_name":"deductive reasoning","score":0.9990000128746033},{"id":"https://openalex.org/keywords/gpt-4v","display_name":"GPT-4V","score":0.777999997138977},{"id":"https://openalex.org/keywords/visual-puzzles","display_name":"visual puzzles","score":0.35199999809265137},{"id":"https://openalex.org/keywords/human-cognitive-processes","display_name":"human cognitive processes","score":0.36800000071525574}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7066121101379395},{"id":"https://openalex.org/C2780878386","wikidata":"https://www.wikidata.org/wiki/Q1659648","display_name":"Visual language","level":2,"score":0.5345432162284851},{"id":"https://openalex.org/C2777508537","wikidata":"https://www.wikidata.org/wiki/Q7936620","display_name":"Visual reasoning","level":2,"score":0.474215030670166},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4682040810585022},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.4465067386627197},{"id":"https://openalex.org/C188147891","wikidata":"https://www.wikidata.org/wiki/Q147638","display_name":"Cognitive science","level":1,"score":0.37219738960266113},{"id":"https://openalex.org/C41895202","wikidata":"https://www.wikidata.org/wiki/Q8162","display_name":"Linguistics","level":1,"score":0.268310010433197},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.12191301584243774},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.18653/v1/2024.findings-acl.962","doi":"https://doi.org/10.18653/v1/2024.findings-acl.962","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.962","pdf_url":"https://aclanthology.org/2024.findings-acl.962.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2403.13315","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.13315","pdf_url":"https://arxiv.org/pdf/2403.13315","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.13315","doi":"https://doi.org/10.48550/arxiv.2403.13315","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.13315","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.18653/v1/2024.findings-acl.962","doi":"https://doi.org/10.18653/v1/2024.findings-acl.962","is_oa":true,"landing_page_url":"https://doi.org/10.18653/v1/2024.findings-acl.962","pdf_url":"https://aclanthology.org/2024.findings-acl.962.pdf","source":{"id":"https://openalex.org/S4363605144","display_name":"Findings of the Association for Computational Linguistics: ACL (Findings of ACL)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Findings of the Association for Computational Linguistics ACL 2024","raw_type":"proceedings-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320322724","display_name":"Ministry of Education, India","ror":"https://ror.org/048xjjh50"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4402684064.pdf","grobid_xml":"https://content.openalex.org/works/W4402684064.grobid-xml"},"referenced_works_count":1,"referenced_works":["https://openalex.org/W4402716477"],"related_works":["https://openalex.org/W7125233878","https://openalex.org/W4411119266","https://openalex.org/W4395065159","https://openalex.org/W7121001577","https://openalex.org/W4392576620","https://openalex.org/W4390041221","https://openalex.org/W4404088350","https://openalex.org/W4416935198","https://openalex.org/W4407128712","https://openalex.org/W4417521014"],"abstract_inverted_index":{"Large":[0],"multimodal":[1,13,60,80,120,170],"models":[2,10,61,109,126,171],"extend":[3],"the":[4,24,115,125,145,166,182],"impressive":[5],"capabilities":[6],"of":[7,30,47,100,148,168],"large":[8,59,79,119,169],"language":[9],"by":[11],"integrating":[12],"understanding":[14],"abilities.However,":[15],"it":[16],"is":[17],"not":[18,87],"clear":[19],"how":[20,173],"they":[21,85,174],"can":[22,175],"emulate":[23,177],"general":[25,40],"intelligence":[26],"and":[27,34,73,138,154,172],"reasoning":[28,116,131,156],"ability":[29],"humans.As":[31],"recognizing":[32],"patterns":[33,64],"abstracting":[35],"concepts":[36],"are":[37,86,150],"key":[38],"to":[39,89,92,162],"intelligence,":[41],"we":[42,57,82,122,160],"introduce":[43],"PUZZLEVQA,":[44],"a":[45,98],"collection":[46],"2000":[48],"puzzle":[49],"instances":[50],"based":[51,65],"on":[52,66,77,102,111,165],"abstract":[53,63,94],"patterns.With":[54],"this":[55,158],"dataset,":[56],"evaluate":[58],"with":[62,127],"fundamental":[67],"concepts,":[68],"including":[69],"colors,":[70],"numbers,":[71],"sizes,":[72],"shapes.Through":[74],"our":[75,112,128],"experiments":[76],"state-of-the-art":[78,108],"models,":[81,121],"find":[83],"that":[84,107,144],"able":[88],"generalize":[90],"well":[91],"simple":[93],"patterns.Notably,":[95],"GPT-4V":[96,149],"achieves":[97],"score":[99],"46.4%":[101],"single-concept":[103],"puzzles,":[104],"which":[105],"shows":[106],"struggle":[110],"dataset.To":[113],"diagnose":[114],"challenges":[117],"in":[118,181],"progressively":[123],"guide":[124],"ground":[129],"truth":[130],"explanations":[132],"for":[133],"visual":[134,152],"perception,":[135],"inductive":[136,155],"reasoning,":[137],"deductive":[139],"reasoning.Our":[140],"systematic":[141],"analysis":[142],"finds":[143],"main":[146],"bottlenecks":[147],"weaker":[151],"perception":[153],"abilities.Through":[157],"work,":[159],"hope":[161],"shed":[163],"light":[164],"limitations":[167],"better":[176],"human":[178],"cognitive":[179],"processes":[180],"future":[183],"1":[184],".":[185]},"counts_by_year":[{"year":2026,"cited_by_count":20},{"year":2025,"cited_by_count":22},{"year":2024,"cited_by_count":8}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-09-21T00:00:00"}