{"id":"https://openalex.org/W4378942271","doi":"https://doi.org/10.48550/arxiv.2305.18354","title":"LLMs and the Abstraction and Reasoning Corpus: Successes, Failures, and the Importance of Object-based Representations","display_name":"LLMs and the Abstraction and Reasoning Corpus: Successes, Failures, and the Importance of Object-based Representations","publication_year":2023,"publication_date":"2023-05-26","ids":{"openalex":"https://openalex.org/W4378942271","doi":"https://doi.org/10.48550/arxiv.2305.18354"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2305.18354","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18354","pdf_url":"https://arxiv.org/pdf/2305.18354","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2305.18354","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100572590","display_name":"Yudong Xu","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Yudong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107145907","display_name":"Wenhao Li","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Wenhao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091641915","display_name":"Pashootan Vaezipoor","orcid":"https://orcid.org/0000-0002-1340-697X","observed_orcids":["https://orcid.org/0000-0002-1340-697X"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vaezipoor, Pashootan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5028174137","display_name":"Scott Sanner","orcid":"https://orcid.org/0000-0001-7984-8394","observed_orcids":["https://orcid.org/0000-0001-7984-8394"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sanner, Scott","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5009528063","display_name":"Elias B. Khalil","orcid":"https://orcid.org/0000-0001-5844-9642","observed_orcids":["https://orcid.org/0000-0001-5844-9642","https://orcid.org/0000-0003-0778-9342"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Khalil, Elias B.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4196999967098236,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4196999967098236,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.23010000586509705,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.09749999642372131,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7749999761581421},{"id":"https://openalex.org/keywords/abstract-thinking","display_name":"abstract thinking","score":0.7699999809265137},{"id":"https://openalex.org/keywords/abstraction-and-reasoning-corpus","display_name":"Abstraction and Reasoning Corpus","score":0.9959999918937683},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9670000076293945},{"id":"https://openalex.org/keywords/object-centric-representation","display_name":"object-centric representation","score":0.8529999852180481},{"id":"https://openalex.org/keywords/failure-analysis","display_name":"failure analysis","score":0.6069999933242798},{"id":"https://openalex.org/keywords/core-knowledge","display_name":"core knowledge","score":0.5170000195503235},{"id":"https://openalex.org/keywords/visual-reasoning","display_name":"visual reasoning","score":0.3050000071525574}],"concepts":[{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.837131917476654},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.6850610375404358},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6749101877212524},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6738777756690979},{"id":"https://openalex.org/C83415579","wikidata":"https://www.wikidata.org/wiki/Q161973","display_name":"Arc (geometry)","level":2,"score":0.6700968146324158},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6337912678718567},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.6201915144920349},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.48629507422447205},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48004066944122314},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.4562067687511444},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.4244942367076874},{"id":"https://openalex.org/C2524010","wikidata":"https://www.wikidata.org/wiki/Q8087","display_name":"Geometry","level":1,"score":0.1148812472820282},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.10144728422164917},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2305.18354","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18354","pdf_url":"https://arxiv.org/pdf/2305.18354","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2305.18354","doi":"https://doi.org/10.48550/arxiv.2305.18354","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2305.18354","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2305.18354","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2305.18354","pdf_url":"https://arxiv.org/pdf/2305.18354","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4379924607","https://openalex.org/W6892041976","https://openalex.org/W4401201037","https://openalex.org/W4387559744","https://openalex.org/W4401110393","https://openalex.org/W7133075281","https://openalex.org/W4410126491","https://openalex.org/W6891866560","https://openalex.org/W7076965178","https://openalex.org/W7223209680"],"abstract_inverted_index":{"Can":[0],"a":[1,17,29,105,109,117,196],"Large":[2],"Language":[3],"Model":[4],"(LLM)":[5],"solve":[6],"simple":[7,197],"abstract":[8,33],"reasoning":[9,34,213],"problems?":[10],"We":[11],"explore":[12],"this":[13,113,148],"broad":[14],"question":[15],"through":[16,158],"systematic":[18],"analysis":[19,78],"of":[20,32,46,61,97,108,124,206],"GPT":[21,216],"on":[22,142,168,175],"the":[23,62,94,98,120,143,166,176,180,193,204],"Abstraction":[24],"and":[25,54,86,135,172,218],"Reasoning":[26],"Corpus":[27],"(ARC),":[28],"representative":[30],"benchmark":[31],"ability":[35],"from":[36],"limited":[37],"examples":[38],"in":[39,163],"which":[40,122],"solutions":[41],"require":[42],"some":[43],"\"core":[44],"knowledge\"":[45],"concepts":[47],"such":[48,191],"as":[49,192],"objects,":[50],"goal":[51],"states,":[52],"counting,":[53],"basic":[55],"geometry.":[56],"GPT-4":[57,182],"solves":[58],"only":[59],"13/50":[60],"most":[63],"straightforward":[64],"ARC":[65,170,198],"tasks":[66,127,171],"when":[67],"using":[68],"textual":[69],"encodings":[70],"for":[71],"their":[72],"two-dimensional":[73],"input-output":[74],"grids.":[75],"Our":[76],"failure":[77],"reveals":[79,202],"that":[80,100,128,155,203],"GPT-4's":[81],"capacity":[82],"to":[83,132,185],"identify":[84],"objects":[85],"reason":[87],"about":[88],"them":[89],"is":[90,156,183],"significantly":[91,210],"influenced":[92],"by":[93],"sequential":[95],"nature":[96],"text":[99,106],"represents":[101],"an":[102,152,159],"object":[103],"within":[104,188],"encoding":[107],"task.":[110],"To":[111,146],"test":[112],"hypothesis,":[114],"we":[115,150],"design":[116],"new":[118],"benchmark,":[119],"1D-ARC,":[121],"consists":[123],"one-dimensional":[125],"(array-like)":[126],"are":[129,220],"more":[130],"conducive":[131],"GPT-based":[133],"reasoning,":[134],"where":[136],"it":[137],"indeed":[138],"performs":[139],"better":[140],"than":[141],"(2D)":[144],"ARC.":[145],"alleviate":[147],"issue,":[149],"propose":[151],"object-based":[153,207],"representation":[154],"obtained":[157],"external":[160],"tool,":[161],"resulting":[162],"nearly":[164],"doubling":[165],"performance":[167],"solved":[169],"near-perfect":[173],"scores":[174],"easier":[177],"1D-ARC.":[178],"Although":[179],"state-of-the-art":[181],"unable":[184],"\"reason\"":[186],"perfectly":[187],"non-language":[189],"domains":[190],"1D-ARC":[194],"or":[195],"subset,":[199],"our":[200],"study":[201],"use":[205],"representations":[208],"can":[209],"improve":[211],"its":[212],"ability.":[214],"Visualizations,":[215],"logs,":[217],"data":[219],"available":[221],"at":[222],"https://khalil-research.github.io/LLM4ARC.":[223]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":4}],"updated_date":"2026-09-26T08:19:03.415552","created_date":"2023-06-01T00:00:00"}