{"id":"https://openalex.org/W4415799112","doi":"https://doi.org/10.52202/079017-0011","title":"Chain of Preference Optimization: Improving Chain-of-Thought Reasoning in LLMs","display_name":"Chain of Preference Optimization: Improving Chain-of-Thought Reasoning in LLMs","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4415799112","doi":"https://doi.org/10.52202/079017-0011"},"language":"en","primary_location":{"id":"doi:10.52202/079017-0011","doi":"https://doi.org/10.52202/079017-0011","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-0011","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2406.09136","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5106715782","display_name":"Xuan Zhang","orcid":"https://orcid.org/0009-0000-7776-4445","observed_orcids":["https://orcid.org/0009-0000-7776-4445"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xuan Zhang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Chao Du","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chao Du","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5031863731","display_name":"Tianyu Pang","orcid":"https://orcid.org/0000-0003-0639-6176","observed_orcids":["https://orcid.org/0000-0003-0639-6176"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tianyu Pang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100596449","display_name":"Qian Liu","orcid":"https://orcid.org/0009-0002-8711-4563","observed_orcids":["https://orcid.org/0009-0002-8711-4563"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qian Liu","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Wei Gao","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wei Gao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5102102207","display_name":"Min Lin","orcid":"https://orcid.org/0009-0002-8984-5669","observed_orcids":["https://orcid.org/0009-0002-8984-5669","https://orcid.org/0000-0001-9584-3374"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Min Lin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":13.5761,"has_fulltext":true,"cited_by_count":89,"citation_normalized_percentile":{"value":0.9916227,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":98,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"333","last_page":"356"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8812999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.8812999725341797,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.06210000067949295,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.013299999758601189,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/tree-of-thoughts","display_name":"tree-of-thought","score":0.9179999828338623},{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7689999938011169},{"id":"https://openalex.org/keywords/tree-search","display_name":"tree search","score":0.5600000023841858},{"id":"https://openalex.org/keywords/preference-data","display_name":"preference data","score":0.6010000109672546},{"id":"https://openalex.org/keywords/fine-tuning","display_name":"fine-tuning","score":0.734000027179718},{"id":"https://openalex.org/keywords/reasoning-paths","display_name":"reasoning paths","score":0.6029999852180481},{"id":"https://openalex.org/keywords/inference-efficiency","display_name":"inference efficiency","score":0.5379999876022339},{"id":"https://openalex.org/keywords/question-answering","display_name":"question answering","score":0.7990000247955322},{"id":"https://openalex.org/keywords/fact-checking","display_name":"fact-checking","score":0.9950000047683716},{"id":"https://openalex.org/keywords/arithmetic-reasoning","display_name":"arithmetic reasoning","score":0.9980000257492065}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5759999752044678},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.412200003862381},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.30489999055862427},{"id":"https://openalex.org/C199185054","wikidata":"https://www.wikidata.org/wiki/Q552299","display_name":"Chain (unit)","level":2,"score":0.3003999888896942},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.28600001335144043},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.28299999237060547},{"id":"https://openalex.org/C79897977","wikidata":"https://www.wikidata.org/wiki/Q5054568","display_name":"Causal chain","level":2,"score":0.2669999897480011},{"id":"https://openalex.org/C162118730","wikidata":"https://www.wikidata.org/wiki/Q1128453","display_name":"Actuarial science","level":1,"score":0.25110000371932983},{"id":"https://openalex.org/C189430467","wikidata":"https://www.wikidata.org/wiki/Q7293293","display_name":"Ranking (information retrieval)","level":2,"score":0.24369999766349792},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.23999999463558197}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.52202/079017-0011","doi":"https://doi.org/10.52202/079017-0011","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-0011","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2406.09136","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.09136","pdf_url":"https://arxiv.org/pdf/2406.09136","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ink.library.smu.edu.sg:sis_research-10881","doi":null,"is_oa":true,"landing_page_url":"https://ink.library.smu.edu.sg/sis_research/9881","pdf_url":null,"source":{"id":"https://openalex.org/S4306401925","display_name":"Singapore Management University Institutional Knowledge (InK) (Singapore Management University)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I79891267","host_organization_name":"Singapore Management University","host_organization_lineage":["https://openalex.org/I79891267"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc-nd","license_id":"https://openalex.org/licenses/cc-by-nc-nd","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"https://openreview.net/pdf?id=2cczgOfMP4","raw_type":"Conference Proceeding Article"},{"id":"doi:10.48550/arxiv.2406.09136","doi":"https://doi.org/10.48550/arxiv.2406.09136","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2406.09136","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2406.09136","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2406.09136","pdf_url":"https://arxiv.org/pdf/2406.09136","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415799112.pdf","grobid_xml":"https://content.openalex.org/works/W4415799112.grobid-xml"},"referenced_works_count":47,"referenced_works":["https://openalex.org/W2767206889","https://openalex.org/W2952862139","https://openalex.org/W2964121744","https://openalex.org/W3102317072","https://openalex.org/W3115947671","https://openalex.org/W3136303386","https://openalex.org/W3138392969","https://openalex.org/W3168867926","https://openalex.org/W4223908421","https://openalex.org/W4281994620","https://openalex.org/W4287121227","https://openalex.org/W4306177491","https://openalex.org/W4311991106","https://openalex.org/W4320167623","https://openalex.org/W4376653732","https://openalex.org/W4378713467","https://openalex.org/W4384918448","https://openalex.org/W4385571841","https://openalex.org/W4386080611","https://openalex.org/W4386081014","https://openalex.org/W4387156632","https://openalex.org/W4387294204","https://openalex.org/W4387561528","https://openalex.org/W4389520103","https://openalex.org/W4389520747","https://openalex.org/W4389911544","https://openalex.org/W4389925942","https://openalex.org/W4390573302","https://openalex.org/W4391047180","https://openalex.org/W4391334917","https://openalex.org/W4392019816","https://openalex.org/W4392539477","https://openalex.org/W4392637187","https://openalex.org/W4393178122","https://openalex.org/W4394973490","https://openalex.org/W4396243737","https://openalex.org/W4399759061","https://openalex.org/W4399911690","https://openalex.org/W4402670732","https://openalex.org/W4402671620","https://openalex.org/W4402703878","https://openalex.org/W4402759247","https://openalex.org/W4404783631","https://openalex.org/W4415797522","https://openalex.org/W4415798413","https://openalex.org/W7076001328","https://openalex.org/W7133199456"],"related_works":["https://openalex.org/W7154924266","https://openalex.org/W7148561217","https://openalex.org/W4403556106","https://openalex.org/W4400376764","https://openalex.org/W4416036976","https://openalex.org/W4415797312","https://openalex.org/W4412944734","https://openalex.org/W7166806345","https://openalex.org/W4404781969","https://openalex.org/W7131126100"],"abstract_inverted_index":{"The":[0,34],"recent":[1],"development":[2],"of":[3,63,102,114,121,146],"chain-of-thought":[4],"(CoT)":[5],"decoding":[6,53],"has":[7],"enabled":[8],"large":[9],"language":[10],"models":[11],"(LLMs)":[12],"to":[13,40,85,110],"generate":[14],"explicit":[15],"logical":[16],"reasoning":[17,44,49,117],"paths":[18,27,50,118],"for":[19],"complex":[20,147],"problem-solving.":[21],"However,":[22],"research":[23],"indicates":[24],"that":[25,51,73,136],"these":[26],"are":[28,108],"not":[29],"always":[30],"deliberate":[31],"and":[32,46,154],"optimal.":[33],"tree-of-thought":[35],"(ToT)":[36],"method":[37],"employs":[38],"tree-searching":[39],"extensively":[41],"explore":[42],"the":[43,61,77,93,115,124,129],"space":[45],"find":[47],"better":[48,89],"CoT":[52,84,116],"might":[54],"overlook.":[55],"This":[56,97],"deliberation,":[57],"however,":[58],"comes":[59],"at":[60,164],"cost":[62],"significantly":[64,138],"increased":[65],"inference":[66,95],"complexity.":[67],"In":[68],"this":[69],"work,":[70],"we":[71],"demonstrate":[72],"fine-tuning":[74],"LLMs":[75,107],"leveraging":[76],"search":[78],"tree":[79],"constructed":[80],"by":[81],"ToT":[82,122],"allows":[83],"achieve":[86],"similar":[87],"or":[88],"performance,":[90],"thereby":[91],"avoiding":[92],"substantial":[94],"burden.":[96],"is":[98,162],"achieved":[99],"through":[100],"Chain":[101],"Preference":[103],"Optimization":[104],"(CPO),":[105],"where":[106],"fine-tuned":[109],"align":[111],"each":[112],"step":[113],"with":[119],"those":[120],"using":[123],"inherent":[125],"preference":[126],"information":[127],"in":[128,142],"tree-search":[130],"process.":[131],"Extensive":[132],"experimental":[133],"results":[134],"show":[135],"CPO":[137],"improves":[139],"LLM":[140],"performance":[141],"solving":[143],"a":[144],"variety":[145],"problems,":[148],"including":[149],"question":[150],"answering,":[151],"fact":[152],"verification,":[153],"arithmetic":[155],"reasoning,":[156],"demonstrating":[157],"its":[158],"effectiveness.":[159],"Our":[160],"code":[161],"available":[163],"https://github.com/sail-sg/CPO.":[165]},"counts_by_year":[{"year":2026,"cited_by_count":45},{"year":2025,"cited_by_count":37},{"year":2024,"cited_by_count":7}],"updated_date":"2026-11-10T11:39:43.021295","created_date":"2025-11-03T00:00:00"}