{"id":"https://openalex.org/W4309591663","doi":"https://doi.org/10.48550/arxiv.2211.10435","title":"PAL: Program-aided Language Models","display_name":"PAL: Program-aided Language Models","publication_year":2022,"publication_date":"2022-11-18","ids":{"openalex":"https://openalex.org/W4309591663","doi":"https://doi.org/10.48550/arxiv.2211.10435"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2211.10435","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.10435","pdf_url":"https://arxiv.org/pdf/2211.10435","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2211.10435","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5060885692","display_name":"Luyu Gao","orcid":"https://orcid.org/0009-0006-5806-3022","observed_orcids":["https://orcid.org/0009-0006-5806-3022"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gao, Luyu","raw_affiliation_strings":["Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5085667632","display_name":"Aman Madaan","orcid":"https://orcid.org/0009-0009-5604-9755","observed_orcids":["https://orcid.org/0009-0009-5604-9755"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Madaan, Aman","raw_affiliation_strings":["Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5113920761","display_name":"Shuyan Zhou","orcid":"https://orcid.org/0009-0002-8868-7996","observed_orcids":["https://orcid.org/0009-0002-8868-7996","https://orcid.org/0009-0003-1532-4542"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Zhou, Shuyan","raw_affiliation_strings":["Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107243336","display_name":"Uri Alon","orcid":"https://orcid.org/0000-0001-5796-9988","observed_orcids":["https://orcid.org/0000-0001-5796-9988"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Alon, Uri","raw_affiliation_strings":["Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100354997","display_name":"Pengfei Liu","orcid":"https://orcid.org/0000-0001-9030-1875","observed_orcids":["https://orcid.org/0000-0001-9030-1875"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liu, Pengfei","raw_affiliation_strings":["Inspired Cognition , USA","Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inspired Cognition , USA","institution_ids":[]},{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5041434096","display_name":"Yiming Yang","orcid":"https://orcid.org/0000-0003-1359-0364","observed_orcids":["https://orcid.org/0000-0003-1359-0364","https://orcid.org/0009-0000-3406-7252","https://orcid.org/0009-0008-4425-3356"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yang, Yiming","raw_affiliation_strings":["Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5009879041","display_name":"Jamie Callan","orcid":"https://orcid.org/0000-0002-1211-7754","observed_orcids":["https://orcid.org/0000-0002-1211-7754"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Callan, Jamie","raw_affiliation_strings":["Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5068811427","display_name":"Graham Neubig","orcid":"https://orcid.org/0000-0002-2072-3789","observed_orcids":["https://orcid.org/0000-0002-2072-3789"]},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Neubig, Graham","raw_affiliation_strings":["Inspired Cognition , USA","Language Tech- nologies Institute , Carnegie Mellon University , USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Inspired Cognition , USA","institution_ids":[]},{"raw_affiliation_string":"Language Tech- nologies Institute , Carnegie Mellon University , USA","institution_ids":["https://openalex.org/I74973139"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I74973139"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":155,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7670999765396118,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7670999765396118,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09269999712705612,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.015200000256299973,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.8510000109672546},{"id":"https://openalex.org/keywords/code-generation","display_name":"code generation","score":0.7129999995231628},{"id":"https://openalex.org/keywords/python-interpreter","display_name":"Python interpreter","score":0.6019999980926514},{"id":"https://openalex.org/keywords/symbolic-reasoning","display_name":"symbolic reasoning","score":0.5609999895095825},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.7139999866485596},{"id":"https://openalex.org/keywords/few-shot-prompting","display_name":"few-shot prompting","score":0.722000002861023},{"id":"https://openalex.org/keywords/chain-of-thought","display_name":"chain-of-thought","score":0.8939999938011169},{"id":"https://openalex.org/keywords/math-word-problems","display_name":"mathematical word problems","score":0.6399999856948853},{"id":"https://openalex.org/keywords/gsm8k","display_name":"GSM8K","score":0.7149999737739563}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7904549837112427},{"id":"https://openalex.org/C122783720","wikidata":"https://www.wikidata.org/wiki/Q183065","display_name":"Interpreter","level":2,"score":0.7587614059448242},{"id":"https://openalex.org/C519991488","wikidata":"https://www.wikidata.org/wiki/Q28865","display_name":"Python (programming language)","level":2,"score":0.6777164936065674},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.6471934914588928},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.5142076015472412},{"id":"https://openalex.org/C169590947","wikidata":"https://www.wikidata.org/wiki/Q47506","display_name":"Compiler","level":2,"score":0.5029935240745544},{"id":"https://openalex.org/C88548561","wikidata":"https://www.wikidata.org/wiki/Q347599","display_name":"sort","level":2,"score":0.4559415578842163},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45348307490348816},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3626371920108795},{"id":"https://openalex.org/C23123220","wikidata":"https://www.wikidata.org/wiki/Q816826","display_name":"Information retrieval","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2211.10435","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.10435","pdf_url":"https://arxiv.org/pdf/2211.10435","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2211.10435","doi":"https://doi.org/10.48550/arxiv.2211.10435","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2211.10435","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"public-domain","license_id":"https://openalex.org/licenses/public-domain","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2211.10435","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.10435","pdf_url":"https://arxiv.org/pdf/2211.10435","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Quality education","score":0.8199999928474426,"id":"https://metadata.un.org/sdg/4"}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2341492732","https://openalex.org/W3187193180","https://openalex.org/W106542691","https://openalex.org/W4287027380","https://openalex.org/W1699080303","https://openalex.org/W1721418433","https://openalex.org/W4297799326","https://openalex.org/W3116064965","https://openalex.org/W2463512549","https://openalex.org/W2246915267"],"abstract_inverted_index":{"Large":[0],"language":[1,113,141,191],"models":[2,101],"(LLMs)":[3],"have":[4],"recently":[5],"demonstrated":[6],"an":[7,197],"impressive":[8],"ability":[9],"to":[10,35,67,110,128,158,206],"perform":[11],"arithmetic":[12,81],"and":[13,80,115,169,177,184,199,242],"symbolic":[14,171],"reasoning":[15,121,179,192,200],"tasks,":[16,193],"when":[17,88],"provided":[18],"with":[19],"a":[20,103,129,133,166,170,202],"few":[21],"examples":[22],"at":[23,70,247],"test":[24],"time":[25],"(\"few-shot":[26],"prompting\").":[27],"Much":[28],"of":[29,61,73,227],"this":[30,71,95,163],"success":[31],"can":[32],"be":[33,68],"attributed":[34],"prompting":[36],"methods":[37],"such":[38,131],"as":[39,55,57,118,132],"\"chain-of-thought'',":[40],"which":[41,233],"employ":[42],"LLMs":[43,65,76],"for":[44,151],"both":[45],"understanding":[46],"the":[47,62,84,89,108,119,125,139,147,152,159,224],"problem":[48,90,142],"description":[49],"by":[50,236],"decomposing":[51,138],"it":[52],"into":[53,143],"steps,":[54,122],"well":[56],"solving":[58,155],"each":[59],"step":[60,127],"problem.":[63],"While":[64],"seem":[66],"adept":[69],"sort":[72],"step-by-step":[74],"decomposition,":[75],"often":[77],"make":[78],"logical":[79],"mistakes":[82],"in":[83],"solution":[85,126],"part,":[86],"even":[87],"is":[91,156],"decomposed":[92],"correctly.":[93],"In":[94,187],"paper,":[96],"we":[97],"present":[98],"Program-Aided":[99],"Language":[100],"(PAL):":[102],"novel":[104],"approach":[105],"that":[106],"uses":[107,234],"LLM":[109,168,198],"read":[111],"natural":[112,140,190],"problems":[114],"generate":[116],"programs":[117],"intermediate":[120],"but":[123],"offloads":[124],"runtime":[130],"Python":[134,203],"interpreter.":[135,160],"With":[136],"PAL,":[137],"runnable":[144],"steps":[145],"remains":[146],"only":[148],"learning":[149],"task":[150],"LLM,":[153],"while":[154],"delegated":[157],"We":[161],"demonstrate":[162],"synergy":[164],"between":[165],"neural":[167],"interpreter":[172,204],"across":[173],"13":[174],"mathematical,":[175],"symbolic,":[176],"algorithmic":[178],"tasks":[180],"from":[181],"BIG-Bench":[182],"Hard":[183],"other":[185],"benchmarks.":[186],"all":[188],"these":[189],"generating":[194],"code":[195,241],"using":[196,201,217],"leads":[205],"more":[207],"accurate":[208],"results":[209],"than":[210],"much":[211],"larger":[212],"models.":[213],"For":[214],"example,":[215],"PAL":[216],"Codex":[218],"achieves":[219],"state-of-the-art":[220],"few-shot":[221],"accuracy":[222],"on":[223],"GSM8K":[225],"benchmark":[226],"math":[228],"word":[229],"problems,":[230],"surpassing":[231],"PaLM-540B":[232],"chain-of-thought":[235],"absolute":[237],"15%":[238],"top-1.":[239],"Our":[240],"data":[243],"are":[244],"publicly":[245],"available":[246],"http://reasonwithpal.com/":[248],".":[249]},"counts_by_year":[{"year":2026,"cited_by_count":3},{"year":2025,"cited_by_count":28},{"year":2024,"cited_by_count":48},{"year":2023,"cited_by_count":76}],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-10T00:00:00"}