{"id":"https://openalex.org/W7161070046","doi":"https://doi.org/10.48550/arxiv.2605.11775","title":"Entropy Polarity in Reinforcement Fine-Tuning: Direction, Asymmetry, and Control","display_name":"Entropy Polarity in Reinforcement Fine-Tuning: Direction, Asymmetry, and Control","publication_year":2026,"publication_date":"2026-05-12","ids":{"openalex":"https://openalex.org/W7161070046","doi":"https://doi.org/10.48550/arxiv.2605.11775"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.11775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.11775","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5136044359","display_name":"Jiazheng Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Jiazheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5134058445","display_name":"Ziche Fu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fu, Ziche","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013223824","display_name":"Junrui Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Junrui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126678477","display_name":"Yunbin Zhao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Yunbin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136076755","display_name":"Yunke Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yunke","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136030296","display_name":"Zhiheng Xi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xi, Zhiheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136078589","display_name":"Long Ma","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ma, Long","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5055648957","display_name":"Chenxin An","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"An, Chenxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136065355","display_name":"Zhihao Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Zhihao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136030945","display_name":"Shichun Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shichun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136031227","display_name":"Dingwei Zhu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhu, Dingwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136070264","display_name":"Shihan Dou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dou, Shihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066867197","display_name":"Shaofan Liu","orcid":"https://orcid.org/0000-0002-2705-7117"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Shaofan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136061149","display_name":"Han Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Han","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136019680","display_name":"Wiggin Zhou","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhou, Wiggin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136016389","display_name":"Aiden Adams","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Adams, Aiden","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136047420","display_name":"Tao Gui","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gui, Tao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136058614","display_name":"Fei Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Fei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136062767","display_name":"Qi Zhang","orcid":"https://orcid.org/0000-0002-0022-9628"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136081304","display_name":"Xuanjing Huang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Huang, Xuanjing","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9164999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9164999723434448,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.01979999989271164,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.00430000014603138,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.6990000009536743},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5634999871253967},{"id":"https://openalex.org/keywords/expansive","display_name":"Expansive","score":0.47540000081062317},{"id":"https://openalex.org/keywords/conditional-entropy","display_name":"Conditional entropy","score":0.3564000129699707},{"id":"https://openalex.org/keywords/entropy-rate","display_name":"Entropy rate","score":0.3472000062465668},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.2939000129699707}],"concepts":[{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.6990000009536743},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5634999871253967},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5013999938964844},{"id":"https://openalex.org/C2780502288","wikidata":"https://www.wikidata.org/wiki/Q28838156","display_name":"Expansive","level":3,"score":0.47540000081062317},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.41019999980926514},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.39879998564720154},{"id":"https://openalex.org/C101721835","wikidata":"https://www.wikidata.org/wiki/Q813908","display_name":"Conditional entropy","level":3,"score":0.3564000129699707},{"id":"https://openalex.org/C125252325","wikidata":"https://www.wikidata.org/wiki/Q1345213","display_name":"Entropy rate","level":4,"score":0.3472000062465668},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3066999912261963},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.30559998750686646},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.2939000129699707},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C42047476","wikidata":"https://www.wikidata.org/wiki/Q6269544","display_name":"Joint quantum entropy","level":3,"score":0.2847000062465668},{"id":"https://openalex.org/C163415756","wikidata":"https://www.wikidata.org/wiki/Q126473","display_name":"Contraction (grammar)","level":2,"score":0.2775999903678894}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.11775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.11775","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.11775","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Policy":[0,147],"entropy":[1,28,43,55,67,72,118,122,157,165],"has":[2],"emerged":[3],"as":[4,167],"a":[5,51,62,74,82,93],"fundamental":[6],"measure":[7],"for":[8,20],"understanding":[9],"and":[10,124,127,155,179,186,201],"controlling":[11],"exploration":[12,136],"in":[13,57,134],"reinforcement":[14],"learning":[15],"with":[16],"verifiable":[17],"rewards":[18],"(RLVR)":[19],"LLMs.":[21],"However,":[22],"existing":[23],"entropy-aware":[24],"methods":[25],"mainly":[26],"regulate":[27],"through":[29,159],"global":[30],"objectives,":[31],"while":[32,137,196],"the":[33,66,163],"token-level":[34,76],"mechanism":[35],"by":[36],"which":[37,150],"sampled":[38,83],"policy":[39,42],"updates":[40],"reshape":[41],"remains":[44],"underexplored.":[45],"In":[46],"this":[47],"work,":[48],"we":[49,115,144],"develop":[50],"theoretical":[52],"framework":[53],"of":[54,65],"mechanics":[56],"RLVR.":[58],"Our":[59],"analysis":[60,90],"yields":[61],"first-order":[63],"approximation":[64],"change,":[68],"giving":[69],"rise":[70],"to":[71],"polarity,":[73],"signed":[75],"quantity":[77],"that":[78,117,125,190],"predicts":[79,121],"how":[80],"much":[81],"update":[84],"expands":[85],"or":[86,110],"contracts":[87],"entropy.":[88],"This":[89],"further":[91],"reveals":[92],"structural":[94],"asymmetry:":[95],"reinforcing":[96],"frequent":[97],"high-probability":[98],"tokens":[99],"triggers":[100],"contraction":[101],"tendencies,":[102],"whereas":[103],"expansive":[104],"tendencies":[105],"typically":[106],"require":[107],"lower-probability":[108],"samples":[109],"stronger":[111],"distributional":[112],"correction.":[113],"Empirically,":[114],"show":[116,189],"polarity":[119,129,153],"reliably":[120],"changes,":[123],"positive":[126],"negative":[128],"branches":[130,154],"play":[131],"complementary":[132],"roles":[133],"preserving":[135],"strengthening":[138],"exploitation.":[139],"Building":[140],"on":[141,183],"these":[142],"insights,":[143],"propose":[145],"Polarity-Aware":[146],"Optimization":[148],"(PAPO),":[149],"preserves":[151],"both":[152],"implements":[156],"control":[158],"advantage":[160],"reweighting.":[161],"With":[162],"empirical":[164],"trajectory":[166],"an":[168],"online":[169],"phase":[170],"signal,":[171],"PAPO":[172,191],"adaptively":[173],"reallocates":[174],"optimization":[175],"pressure":[176],"between":[177],"entropy-expanding":[178],"entropy-contracting":[180],"updates.":[181],"Experiments":[182],"mathematical":[184],"reasoning":[185],"agentic":[187],"benchmarks":[188],"consistently":[192],"outperforms":[193],"competitive":[194],"baselines,":[195],"delivering":[197],"superior":[198],"training":[199],"efficiency":[200],"substantial":[202],"reward":[203],"improvements.":[204]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-14T00:00:00"}