{"id":"https://openalex.org/W4391833261","doi":"https://doi.org/10.48550/arxiv.2402.08115","title":"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks","display_name":"On the Self-Verification Limitations of Large Language Models on Reasoning and Planning Tasks","publication_year":2024,"publication_date":"2024-02-12","ids":{"openalex":"https://openalex.org/W4391833261","doi":"https://doi.org/10.48550/arxiv.2402.08115"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2402.08115","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.08115","pdf_url":"https://arxiv.org/pdf/2402.08115","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2402.08115","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5093104363","display_name":"Kaya Stechly","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Stechly, Kaya","raw_affiliation_strings":["Arizona State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arizona State University","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017044794","display_name":"Karthik Valmeekam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valmeekam, Karthik","raw_affiliation_strings":["Arizona State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arizona State University","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023323170","display_name":"Subbarao Kambhampati B","orcid":"https://orcid.org/0000-0002-9069-0265","observed_orcids":["https://orcid.org/0000-0002-9069-0265"]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kambhampati, Subbarao","raw_affiliation_strings":["Arizona State University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Arizona State University","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I55732556"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4081999957561493,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.4081999957561493,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.3986999988555908,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.09889999777078629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.953000009059906},{"id":"https://openalex.org/keywords/self-criticism","display_name":"self-criticism","score":0.8870000243186951},{"id":"https://openalex.org/keywords/iterative-prompting","display_name":"iterative prompting","score":0.7710000276565552},{"id":"https://openalex.org/keywords/verification","display_name":"verification","score":0.3140000104904175},{"id":"https://openalex.org/keywords/reasoning","display_name":"reasoning","score":0.5720000267028809},{"id":"https://openalex.org/keywords/planning","display_name":"planning","score":0.9610000252723694},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9290000200271606},{"id":"https://openalex.org/keywords/graph-coloring","display_name":"graph coloring","score":0.9890000224113464},{"id":"https://openalex.org/keywords/strips-planning","display_name":"STRIPS planning","score":0.996999979019165},{"id":"https://openalex.org/keywords/performance-collapse","display_name":"performance collapse","score":0.5049999952316284},{"id":"https://openalex.org/keywords/computational-complexity","display_name":"computational complexity","score":0.4009999930858612}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6573631763458252},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.45135462284088135},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4014994204044342}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2402.08115","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.08115","pdf_url":"https://arxiv.org/pdf/2402.08115","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2402.08115","doi":"https://doi.org/10.48550/arxiv.2402.08115","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2402.08115","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2402.08115","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2402.08115","pdf_url":"https://arxiv.org/pdf/2402.08115","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4391833261.pdf","grobid_xml":"https://content.openalex.org/works/W4391833261.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W4387635374","https://openalex.org/W4387838288","https://openalex.org/W7117857712","https://openalex.org/W4389524159","https://openalex.org/W4392517661","https://openalex.org/W7117989186","https://openalex.org/W7133237860","https://openalex.org/W7106268488","https://openalex.org/W7124907532","https://openalex.org/W4416456107"],"abstract_inverted_index":{"There":[0],"has":[1,27],"been":[2,28],"considerable":[3],"divergence":[4],"of":[5,11,34,68,108,114,124,127,133,167,180,214,217],"opinion":[6],"on":[7,63],"the":[8,17,64,87,106,112,125,144,165,181,215],"reasoning":[9,21,115],"abilities":[10],"Large":[12],"Language":[13],"Models":[14],"(LLMs).":[15],"While":[16],"initial":[18],"optimism":[19],"that":[20,46,66,89,205],"might":[22],"emerge":[23],"automatically":[24],"with":[25,143,151,192,198,208],"scale":[26],"tempered":[29],"thanks":[30],"to":[31,38,84,86,103],"a":[32,42,120,209],"slew":[33],"counterexamples--ranging":[35],"from":[36,78],"multiplication":[37],"simple":[39],"planning--there":[40],"persists":[41],"wide":[43],"spread":[44],"belief":[45,60],"LLMs":[47,85],"can":[48,177],"self-critique":[49,193],"and":[50,116,137,150,174,194],"improve":[51],"their":[52],"own":[53,148],"solutions":[54],"in":[55,111,129],"an":[56,152],"iterative":[57,109],"fashion.":[58],"This":[59],"seemingly":[61],"rests":[62],"assumption":[65],"verification":[67],"correctness":[69],"should":[70,81],"be":[71,82],"easier":[72],"than":[73],"generation--a":[74],"rather":[75],"classical":[76],"argument":[77],"computational":[79],"complexity--which":[80],"irrelevant":[83],"extent":[88],"what":[90],"they":[91],"are":[92],"doing":[93],"is":[94],"approximate":[95],"retrieval.":[96],"In":[97,159],"this":[98],"paper,":[99],"we":[100,162,176],"set":[101],"out":[102],"systematically":[104],"investigate":[105],"effectiveness":[107],"prompting":[110],"context":[113],"planning.":[117,139],"We":[118,140,187,202],"present":[119],"principled":[121],"empirical":[122],"study":[123],"performance":[126,190,196],"GPT-4":[128],"three":[130],"domains:":[131],"Game":[132],"24,":[134],"Graph":[135],"Coloring,":[136],"STRIPS":[138],"experiment":[141],"both":[142],"model":[145],"critiquing":[146],"its":[147],"answers":[149],"external":[153,200],"correct":[154],"reasoner":[155],"verifying":[156],"proposed":[157],"solutions.":[158],"each":[160],"case,":[161],"analyze":[163],"whether":[164,175],"content":[166],"criticisms":[168],"actually":[169],"affects":[170],"bottom":[171],"line":[172],"performance,":[173],"ablate":[178],"elements":[179],"augmented":[182],"system":[183],"without":[184],"losing":[185],"performance.":[186],"observe":[188],"significant":[189,195],"collapse":[191],"gains":[197],"sound":[199,210],"verification.":[201],"also":[203],"note":[204],"merely":[206],"re-prompting":[207],"verifier":[211],"maintains":[212],"most":[213],"benefits":[216],"more":[218],"involved":[219],"setups.":[220]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":2}],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2024-02-15T00:00:00"}