{"id":"https://openalex.org/W4387635374","doi":"https://doi.org/10.48550/arxiv.2310.08118","title":"Can Large Language Models Really Improve by Self-critiquing Their Own Plans?","display_name":"Can Large Language Models Really Improve by Self-critiquing Their Own Plans?","alternate_titles":[],"publication_year":2023,"publication_date":"2023-10-12","ids":{"openalex":"https://openalex.org/W4387635374","doi":"https://doi.org/10.48550/arxiv.2310.08118"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2310.08118","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.08118","pdf_url":"https://arxiv.org/pdf/2310.08118","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4387635374"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2310.08118","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5017044794","display_name":"Karthik Valmeekam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Valmeekam, Karthik","raw_affiliation_strings":["School of Computing & AI Arizona State University Tempe "],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing & AI Arizona State University Tempe ","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003839565","display_name":"Matthew Marquez","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Marquez, Matthew","raw_affiliation_strings":["School of Computing & AI Arizona State University , Tempe"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing & AI Arizona State University , Tempe","institution_ids":["https://openalex.org/I55732556"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5023323170","display_name":"Subbarao Kambhampati","orcid":"https://orcid.org/0000-0002-9069-0265","observed_orcids":["https://orcid.org/0000-0002-9069-0265"]},"institutions":[{"id":"https://openalex.org/I55732556","display_name":"Arizona State University","ror":"https://ror.org/03efmqc40","country_code":"US","type":"education","lineage":["https://openalex.org/I55732556"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kambhampati, Subbarao","raw_affiliation_strings":["School of Computing & AI Arizona State University , Tempe"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Computing & AI Arizona State University , Tempe","institution_ids":["https://openalex.org/I55732556"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I55732556"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":131,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.0003000000142492354,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.9750000238418579},{"id":"https://openalex.org/keywords/self-criticism","display_name":"self-criticism","score":0.9670000076293945},{"id":"https://openalex.org/keywords/plan-verification","display_name":"plan verification","score":0.9330000281333923},{"id":"https://openalex.org/keywords/automated-planning","display_name":"automated planning","score":0.7099999785423279},{"id":"https://openalex.org/keywords/plan-generation","display_name":"plan generation","score":0.8069999814033508},{"id":"https://openalex.org/keywords/iterative-reasoning","display_name":"iterative reasoning","score":0.628000020980835},{"id":"https://openalex.org/keywords/feedback-levels","display_name":"feedback levels","score":0.5350000262260437},{"id":"https://openalex.org/keywords/false-positive-results","display_name":"false positives","score":0.7509999871253967},{"id":"https://openalex.org/keywords/gpt-4","display_name":"GPT-4","score":0.9100000262260437}],"concepts":[{"id":"https://openalex.org/C2776505523","wikidata":"https://www.wikidata.org/wiki/Q4785468","display_name":"Plan (archaeology)","level":2,"score":0.740096390247345},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6792493462562561},{"id":"https://openalex.org/C2779343474","wikidata":"https://www.wikidata.org/wiki/Q3109175","display_name":"Context (archaeology)","level":2,"score":0.6383459568023682},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5635470747947693},{"id":"https://openalex.org/C64869954","wikidata":"https://www.wikidata.org/wiki/Q1859747","display_name":"False positive paradox","level":2,"score":0.44884076714515686},{"id":"https://openalex.org/C2779982483","wikidata":"https://www.wikidata.org/wiki/Q6094420","display_name":"Iterative refinement","level":2,"score":0.43703192472457886},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.39797908067703247},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.33574825525283813},{"id":"https://openalex.org/C115903868","wikidata":"https://www.wikidata.org/wiki/Q80993","display_name":"Software engineering","level":1,"score":0.3232530355453491},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.20164349675178528},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C166957645","wikidata":"https://www.wikidata.org/wiki/Q23498","display_name":"Archaeology","level":1,"score":0.0},{"id":"https://openalex.org/C95457728","wikidata":"https://www.wikidata.org/wiki/Q309","display_name":"History","level":0,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2310.08118","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.08118","pdf_url":"https://arxiv.org/pdf/2310.08118","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2310.08118","doi":"https://doi.org/10.48550/arxiv.2310.08118","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2310.08118","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2310.08118","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2310.08118","pdf_url":"https://arxiv.org/pdf/2310.08118","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4387635374.pdf","grobid_xml":"https://content.openalex.org/works/W4387635374.grobid-xml"},"referenced_works_count":12,"referenced_works":["https://openalex.org/W2149361370","https://openalex.org/W2164457256","https://openalex.org/W4327810158","https://openalex.org/W4378509427","https://openalex.org/W4389520333","https://openalex.org/W4389524159","https://openalex.org/W7133191918","https://openalex.org/W7133209886","https://openalex.org/W7133210150","https://openalex.org/W7133222210","https://openalex.org/W7133236450","https://openalex.org/W7133237860"],"related_works":["https://openalex.org/W4391833261","https://openalex.org/W7117989186","https://openalex.org/W7133237860","https://openalex.org/W4392517661","https://openalex.org/W4321011818","https://openalex.org/W7127683255","https://openalex.org/W4405432542","https://openalex.org/W7220230041","https://openalex.org/W7138857416","https://openalex.org/W4406786737"],"abstract_inverted_index":{"There":[0],"have":[1],"been":[2],"widespread":[3],"claims":[4],"about":[5],"Large":[6],"Language":[7],"Models":[8],"(LLMs)":[9],"being":[10],"able":[11],"to":[12,37,107,115],"successfully":[13],"verify":[14],"or":[15,146],"self-critique":[16],"their":[17],"candidate":[18],"solutions":[19],"in":[20,23,31,46,125,164],"reasoning":[21],"problems":[22],"an":[24],"iterative":[25,167],"mode.":[26],"Intrigued":[27],"by":[28],"those":[29],"claims,":[30],"this":[32],"paper":[33],"we":[34],"set":[35],"out":[36],"investigate":[38],"the":[39,47,67,74,82,122,136,140,160],"verification/self-critiquing":[40],"abilities":[41],"of":[42,49,76,84,132,142,162],"large":[43],"language":[44],"models":[45],"context":[48],"planning.":[50],"We":[51,65],"evaluate":[52],"a":[53,93,129,165],"planning":[54,170],"system":[55,89,127],"that":[56,104,126],"employs":[57],"LLMs":[58,163],"for":[59,96,169],"both":[60,97],"plan":[61,79,109,152],"generation":[62,98,110],"and":[63,81,99,121],"verification.":[64],"assess":[66],"verifier":[68],"LLM's":[69],"performance":[70],"against":[71],"ground-truth":[72],"verification,":[73,100],"impact":[75,150],"self-critiquing":[77,105],"on":[78,88,151,159],"generation,":[80],"influence":[83],"varying":[85],"feedback":[86],"levels":[87],"performance.":[90],"Using":[91],"GPT-4,":[92],"state-of-the-art":[94],"LLM,":[95],"our":[101,155],"findings":[102],"reveal":[103],"appears":[106],"diminish":[108],"performance,":[111],"especially":[112],"when":[113],"compared":[114],"systems":[116],"with":[117],"external,":[118],"sound":[119],"verifiers":[120,124],"LLM":[123],"produce":[128],"notable":[130],"number":[131],"false":[133],"positives,":[134],"compromising":[135],"system's":[137],"reliability.":[138],"Additionally,":[139],"nature":[141],"feedback,":[143],"whether":[144],"binary":[145],"detailed,":[147],"showed":[148],"minimal":[149],"generation.":[153],"Collectively,":[154],"results":[156],"cast":[157],"doubt":[158],"effectiveness":[161],"self-critiquing,":[166],"framework":[168],"tasks.":[171]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4387635374","counts_by_year":[{"year":2026,"cited_by_count":23},{"year":2025,"cited_by_count":48},{"year":2024,"cited_by_count":54},{"year":2023,"cited_by_count":6}],"updated_date":"2026-11-18T13:11:19.462570","created_date":"2023-10-14T00:00:00"}
