{"id":"https://openalex.org/W4403363385","doi":"https://doi.org/10.48550/arxiv.2410.06667","title":"Large Language Models as Code Executors: An Exploratory Study","display_name":"Large Language Models as Code Executors: An Exploratory Study","publication_year":2024,"publication_date":"2024-10-09","ids":{"openalex":"https://openalex.org/W4403363385","doi":"https://doi.org/10.48550/arxiv.2410.06667"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.06667","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.06667","pdf_url":"https://arxiv.org/pdf/2410.06667","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.06667","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043763521","display_name":"Chenyang Lyu","orcid":"https://orcid.org/0009-0002-6733-5879","observed_orcids":["https://orcid.org/0009-0002-6733-5879"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lyu, Chenyang","raw_affiliation_strings":["University of Artificial Intelligence"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Artificial Intelligence","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Yan, Lecheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I96908189","display_name":"Xinjiang University","ror":"https://ror.org/059gw8r13","country_code":"CN","type":"education","lineage":["https://openalex.org/I96908189"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yan, Lecheng","raw_affiliation_strings":["Xinjiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xinjiang University","institution_ids":["https://openalex.org/I96908189"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5111396472","display_name":"Rui Xing","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"Xing, Rui","raw_affiliation_strings":["University of Melbourne"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Melbourne","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Li, Wenxi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Wenxi","raw_affiliation_strings":["Shanghai Jiao Tong University","Tsinghua University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Shanghai Jiao Tong University","institution_ids":["https://openalex.org/I183067930"]},{"raw_affiliation_string":"Tsinghua University","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086381528","display_name":"Younes Samih","orcid":"https://orcid.org/0000-0002-0485-7920","observed_orcids":["https://orcid.org/0000-0002-0485-7920"]},"institutions":[{"id":"https://openalex.org/I1341412227","display_name":"IBM (United States)","ror":"https://ror.org/05hh8d621","country_code":"US","type":"company","lineage":["https://openalex.org/I1341412227"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samih, Younes","raw_affiliation_strings":["IBM Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"IBM Research","institution_ids":["https://openalex.org/I1341412227"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5071808662","display_name":"Tianbo Ji","orcid":"https://orcid.org/0000-0003-0143-6220","observed_orcids":["https://orcid.org/0000-0003-0143-6220"]},"institutions":[{"id":"https://openalex.org/I199305430","display_name":"Nantong University","ror":"https://ror.org/02afcvw97","country_code":"CN","type":"education","lineage":["https://openalex.org/I199305430"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ji, Tianbo","raw_affiliation_strings":["Nantong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Nantong University","institution_ids":["https://openalex.org/I199305430"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5088191810","display_name":"Longyue Wang","orcid":"https://orcid.org/0000-0002-9062-6183","observed_orcids":["https://orcid.org/0000-0002-9062-6183","https://orcid.org/0009-0009-4675-0417"]},"institutions":[{"id":"https://openalex.org/I45928872","display_name":"Alibaba Group (China)","ror":"https://ror.org/00k642b80","country_code":"CN","type":"company","lineage":["https://openalex.org/I45928872"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wang, Longyue","raw_affiliation_strings":["Alibaba"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Alibaba","institution_ids":["https://openalex.org/I45928872"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":18,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9409000277519226,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9409000277519226,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.031700000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.006000000052154064,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8980000019073486},{"id":"https://openalex.org/keywords/program-execution","display_name":"program execution","score":0.9580000042915344},{"id":"https://openalex.org/keywords/chain-of-thought-prompting","display_name":"chain-of-thought prompting","score":0.8090000152587891},{"id":"https://openalex.org/keywords/program-synthesis","display_name":"program synthesis","score":0.6800000071525574},{"id":"https://openalex.org/keywords/gpt-4o","display_name":"GPT-4o","score":0.8009999990463257},{"id":"https://openalex.org/keywords/openai-o1","display_name":"OpenAI o1","score":0.8069999814033508},{"id":"https://openalex.org/keywords/deepseek","display_name":"DeepSeek","score":0.8920000195503235},{"id":"https://openalex.org/keywords/gpt-3-5","display_name":"GPT-3.5","score":0.8140000104904175}],"concepts":[{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6399232745170593},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6124036312103271},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5525050163269043},{"id":"https://openalex.org/C85973986","wikidata":"https://www.wikidata.org/wiki/Q1091731","display_name":"Exploratory research","level":2,"score":0.5025434494018555},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.17406177520751953},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.12946677207946777},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.06667","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.06667","pdf_url":"https://arxiv.org/pdf/2410.06667","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.06667","doi":"https://doi.org/10.48550/arxiv.2410.06667","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.06667","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.06667","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.06667","pdf_url":"https://arxiv.org/pdf/2410.06667","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403363385.pdf","grobid_xml":"https://content.openalex.org/works/W4403363385.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W3177813494","https://openalex.org/W4327810158","https://openalex.org/W4386185600","https://openalex.org/W4389520065","https://openalex.org/W4391272793","https://openalex.org/W4392669753","https://openalex.org/W4400023054","https://openalex.org/W4403796339","https://openalex.org/W7125135101"],"related_works":["https://openalex.org/W7125493689","https://openalex.org/W7164863649","https://openalex.org/W4416034577","https://openalex.org/W7117124031","https://openalex.org/W4380993527","https://openalex.org/W4388717686","https://openalex.org/W4413349688","https://openalex.org/W4415131828","https://openalex.org/W4416062362","https://openalex.org/W4386185625"],"abstract_inverted_index":{"The":[0],"capabilities":[1,29],"of":[2,27,49,125,131,137,144,153,163,181],"Large":[3],"Language":[4],"Models":[5],"(LLMs)":[6],"have":[7],"significantly":[8],"evolved,":[9],"extending":[10],"from":[11],"natural":[12],"language":[13],"processing":[14],"to":[15,30,36,40,60,73],"complex":[16,182],"tasks":[17],"like":[18],"code":[19,38,52,55,98,117],"understanding":[20],"and":[21,65,87,139,178],"generation.":[22],"We":[23,69],"expand":[24],"the":[25,42,47,61,71,90,123,134,150,160,170,179],"scope":[26],"LLMs'":[28],"a":[31],"broader":[32],"context,":[33],"using":[34],"LLMs":[35,50,164],"execute":[37],"snippets":[39,56,118],"obtain":[41],"output.":[43],"This":[44],"paper":[45],"pioneers":[46],"exploration":[48],"as":[51],"executors,":[53],"where":[54],"are":[57,67,70],"directly":[58],"fed":[59],"models":[62,127],"for":[63,172],"execution,":[64,99],"outputs":[66],"returned.":[68],"first":[72],"comprehensively":[74],"examine":[75],"this":[76],"feasibility":[77],"across":[78],"various":[79],"LLMs,":[80],"including":[81],"OpenAI's":[82],"o1,":[83],"GPT-4o,":[84],"GPT-3.5,":[85],"DeepSeek,":[86],"Qwen-Coder.":[88],"Notably,":[89],"o1":[91],"model":[92],"achieved":[93],"over":[94],"90%":[95],"accuracy":[96,104,124],"in":[97,165,175],"while":[100],"others":[101],"demonstrated":[102],"lower":[103],"levels.":[105],"Furthermore,":[106],"we":[107],"introduce":[108],"an":[109,129,140],"Iterative":[110],"Instruction":[111],"Prompting":[112],"(IIP)":[113],"technique":[114],"that":[115],"processes":[116],"line":[119],"by":[120,128],"line,":[121],"enhancing":[122],"weaker":[126],"average":[130,142],"7.22%":[132],"(with":[133,149],"highest":[135,151],"improvement":[136,143,152],"18.96%)":[138],"absolute":[141],"3.86%":[145],"against":[146],"CoT":[147],"prompting":[148],"19.46%).":[154],"Our":[155],"study":[156],"not":[157],"only":[158],"highlights":[159],"transformative":[161],"potential":[162],"coding":[166],"but":[167],"also":[168],"lays":[169],"groundwork":[171],"future":[173],"advancements":[174],"automated":[176],"programming":[177],"completion":[180],"tasks.":[183]},"counts_by_year":[{"year":2026,"cited_by_count":8},{"year":2025,"cited_by_count":10}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-13T00:00:00"}