{"id":"https://openalex.org/W7127069098","doi":"https://doi.org/10.48550/arxiv.2601.22648","title":"UCPO: Uncertainty-Aware Policy Optimization","display_name":"UCPO: Uncertainty-Aware Policy Optimization","publication_year":2026,"publication_date":"2026-01-30","ids":{"openalex":"https://openalex.org/W7127069098","doi":"https://doi.org/10.48550/arxiv.2601.22648"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2601.22648","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2601.22648","pdf_url":"https://arxiv.org/pdf/2601.22648","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2601.22648","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5124827954","display_name":"Xianzhou Zeng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zeng, Xianzhou","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124840818","display_name":"Jing Huang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huang, Jing","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100524377","display_name":"Chunmei Xie","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xie, Chunmei","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5086803311","display_name":"Gongrui Nan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nan, Gongrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124811365","display_name":"Siye Chen","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Chen, Siye","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124793786","display_name":"Mengyu Lu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Lu, Mengyu","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124833487","display_name":"Weiqi Xiong","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiong, Weiqi","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137226812","display_name":"Qixuan Zhou","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhou, Qixuan","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021853581","display_name":"Junhao Zhang","orcid":"https://orcid.org/0009-0004-0990-672X","observed_orcids":["https://orcid.org/0009-0004-0990-672X"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhang, Junhao","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5124845896","display_name":"Qiang Zhu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zhu, Qiang","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100348451","display_name":"Yadong Li","orcid":"https://orcid.org/0009-0007-6167-2117","observed_orcids":["https://orcid.org/0009-0007-6167-2117","https://orcid.org/0000-0002-9882-6788"]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Yadong","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5124792644","display_name":"Xingzhong Xu","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xu, Xingzhong","raw_affiliation_strings":["Zhejiang University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zhejiang University","institution_ids":["https://openalex.org/I76130692"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8345000147819519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8345000147819519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.03530000150203705,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.031199999153614044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-models","display_name":"large language models","score":0.7059999704360962},{"id":"https://openalex.org/keywords/reward-hacking","display_name":"reward hacking","score":0.796999990940094},{"id":"https://openalex.org/keywords/overconfidence","display_name":"overconfidence","score":0.6669999957084656},{"id":"https://openalex.org/keywords/mathematical-reasoning","display_name":"mathematical reasoning","score":0.7229999899864197},{"id":"https://openalex.org/keywords/trustworthy-ai","display_name":"trustworthy AI","score":0.5170000195503235}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6674000024795532},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.49230000376701355},{"id":"https://openalex.org/C51110983","wikidata":"https://www.wikidata.org/wiki/Q16503490","display_name":"Overconfidence effect","level":2,"score":0.4869999885559082},{"id":"https://openalex.org/C165838908","wikidata":"https://www.wikidata.org/wiki/Q736777","display_name":"Calibration","level":2,"score":0.399399995803833},{"id":"https://openalex.org/C43214815","wikidata":"https://www.wikidata.org/wiki/Q7310987","display_name":"Reliability (semiconductor)","level":3,"score":0.39500001072883606},{"id":"https://openalex.org/C96640997","wikidata":"https://www.wikidata.org/wiki/Q5163004","display_name":"Conservatism","level":3,"score":0.34700000286102295},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.329800009727478},{"id":"https://openalex.org/C205606062","wikidata":"https://www.wikidata.org/wiki/Q5249645","display_name":"Decoupling (probability)","level":2,"score":0.3197000026702881},{"id":"https://openalex.org/C153701036","wikidata":"https://www.wikidata.org/wiki/Q659974","display_name":"Trustworthiness","level":2,"score":0.30649998784065247},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2863999903202057},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C39920170","wikidata":"https://www.wikidata.org/wiki/Q693083","display_name":"Soundness","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C55660270","wikidata":"https://www.wikidata.org/wiki/Q5164377","display_name":"Constrained optimization","level":2,"score":0.2655999958515167},{"id":"https://openalex.org/C48372109","wikidata":"https://www.wikidata.org/wiki/Q3913","display_name":"Binary number","level":2,"score":0.2630999982357025},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2621000111103058},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.26190000772476196},{"id":"https://openalex.org/C2781067378","wikidata":"https://www.wikidata.org/wiki/Q17027399","display_name":"Interpretability","level":2,"score":0.25}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2601.22648","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2601.22648","pdf_url":"https://arxiv.org/pdf/2601.22648","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2601.22648","doi":"https://doi.org/10.48550/arxiv.2601.22648","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2601.22648","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2601.22648","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2601.22648","pdf_url":"https://arxiv.org/pdf/2601.22648","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7127069098.pdf","grobid_xml":"https://content.openalex.org/works/W7127069098.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W3083410900","https://openalex.org/W3123791752","https://openalex.org/W4321594373","https://openalex.org/W4389518686","https://openalex.org/W4389518688","https://openalex.org/W4391244604","https://openalex.org/W4391631327","https://openalex.org/W4409049124","https://openalex.org/W4414281281","https://openalex.org/W4414427575","https://openalex.org/W4414685036","https://openalex.org/W4414896318","https://openalex.org/W4414928251","https://openalex.org/W4414939355","https://openalex.org/W4416243481","https://openalex.org/W4416258665","https://openalex.org/W4416612974","https://openalex.org/W4416649973","https://openalex.org/W4417080162","https://openalex.org/W4417258166","https://openalex.org/W7084062577","https://openalex.org/W7120177137","https://openalex.org/W7155201584","https://openalex.org/W7196953391"],"related_works":["https://openalex.org/W4392680698","https://openalex.org/W7165375242","https://openalex.org/W4417300720","https://openalex.org/W4403821502","https://openalex.org/W4414809468","https://openalex.org/W7166851443","https://openalex.org/W4416095002","https://openalex.org/W7131731643","https://openalex.org/W7134941442","https://openalex.org/W4390528609"],"abstract_inverted_index":{"The":[0],"key":[1],"to":[2,38,90,116],"building":[3],"trustworthy":[4],"large":[5],"language":[6],"models":[7],"(LLMs)":[8],"lies":[9],"in":[10,22,67,113,124],"endowing":[11],"them":[12],"with":[13],"inherent":[14],"uncertainty":[15,44,111],"expression":[16],"capabilities,":[17],"thereby":[18,99],"mitigating":[19],"overconfident":[20],"errors":[21],"high-stakes":[23],"applications.":[24],"However,":[25],"existing":[26],"RL":[27,69],"paradigms":[28,70],"such":[29],"as":[30],"GRPO":[31],"often":[32],"suffer":[33],"from":[34],"Advantage":[35,88],"Bias":[36],"due":[37],"binary":[39],"decision":[40],"spaces":[41],"and":[42,65,92,96,119,127],"static":[43],"rewards,":[45,73],"inducing":[46],"either":[47],"excessive":[48],"conservatism":[49],"or":[50],"overconfidence.":[51],"To":[52],"tackle":[53],"this":[54,56],"challenge,":[55],"paper":[57],"unveils":[58],"the":[59,79,135,140,143],"root":[60],"causes":[61],"of":[62,142],"reward":[63,136],"hacking":[64],"overconfidence":[66],"current":[68],"incorporating":[71],"uncertainty-based":[72],"based":[74],"on":[75],"which":[76],"we":[77],"propose":[78],"UnCertainty-Aware":[80],"Policy":[81],"Optimization":[82],"(UCPO)":[83],"framework.":[84],"UCPO":[85,132],"employs":[86],"Ternary":[87],"Decoupling":[89],"separate":[91],"independently":[93],"normalize":[94],"deterministic":[95],"uncertain":[97],"rollouts,":[98],"eliminating":[100],"advantage":[101],"bias.":[102],"Furthermore,":[103],"a":[104],"Dynamic":[105],"Uncertainty":[106],"Reward":[107],"Adjustment":[108],"mechanism":[109],"adapts":[110],"weights":[112],"real-time":[114],"according":[115],"model":[117,144],"evolution":[118],"instance":[120],"difficulty.":[121],"Experimental":[122],"results":[123],"mathematical":[125],"reasoning":[126],"general":[128],"tasks":[129],"demonstrate":[130],"that":[131],"effectively":[133],"resolves":[134],"imbalance,":[137],"significantly":[138],"improving":[139],"reliability":[141],"beyond":[145],"their":[146],"knowledge":[147],"boundaries.":[148]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-02-03T00:00:00"}