{"id":"https://openalex.org/W4416429290","doi":"https://doi.org/10.1109/tnnls.2025.3626050","title":"Universal Stabilization for Maximum Entropy Optimization in Reinforcement Learning","display_name":"Universal Stabilization for Maximum Entropy Optimization in Reinforcement Learning","publication_year":2025,"publication_date":"2025-11-20","ids":{"openalex":"https://openalex.org/W4416429290","doi":"https://doi.org/10.1109/tnnls.2025.3626050","pmid":"https://pubmed.ncbi.nlm.nih.gov/41264432"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2025.3626050","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3626050","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100371757","display_name":"Xing Chen","orcid":"https://orcid.org/0000-0001-5685-8506"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xing Chen","raw_affiliation_strings":["School of Artificial Intelligence, Jilin University, Changchun, China"],"raw_orcid":"https://orcid.org/0000-0001-5685-8506","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Jilin University, Changchun, China","institution_ids":["https://openalex.org/I194450716"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058350437","display_name":"Yewen Li","orcid":"https://orcid.org/0009-0008-0073-123X"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Yewen Li","raw_affiliation_strings":["College of Computing and Data Science, Nanyang Technological University, Jurong West, Singapore"],"raw_orcid":"https://orcid.org/0009-0008-0073-123X","affiliations":[{"raw_affiliation_string":"College of Computing and Data Science, Nanyang Technological University, Jurong West, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5043782065","display_name":"Xiaofeng Cao","orcid":"https://orcid.org/0000-0002-7391-0334"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xiaofeng Cao","raw_affiliation_strings":["School of Artificial Intelligence, Jilin University, Changchun, China"],"raw_orcid":"https://orcid.org/0000-0002-7391-0334","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Jilin University, Changchun, China","institution_ids":["https://openalex.org/I194450716"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5108294333","display_name":"Hechang Chen","orcid":"https://orcid.org/0000-0001-7835-9556"},"institutions":[{"id":"https://openalex.org/I194450716","display_name":"Jilin University","ror":"https://ror.org/00js3aw79","country_code":"CN","type":"education","lineage":["https://openalex.org/I194450716"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hechang Chen","raw_affiliation_strings":["School of Artificial Intelligence, Jilin University, Changchun, China"],"raw_orcid":"https://orcid.org/0000-0001-7835-9556","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Jilin University, Changchun, China","institution_ids":["https://openalex.org/I194450716"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050876115","display_name":"Hengshuai Yao","orcid":"https://orcid.org/0000-0003-1258-1845"},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Hengshuai Yao","raw_affiliation_strings":["Department of Computing Science, University of Alberta, Edmonton, Canada"],"raw_orcid":"https://orcid.org/0000-0003-1258-1845","affiliations":[{"raw_affiliation_string":"Department of Computing Science, University of Alberta, Edmonton, Canada","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5017743551","display_name":"Bo An","orcid":"https://orcid.org/0000-0002-7064-7438"},"institutions":[{"id":"https://openalex.org/I172675005","display_name":"Nanyang Technological University","ror":"https://ror.org/02e7b5302","country_code":"SG","type":"education","lineage":["https://openalex.org/I172675005"]}],"countries":["SG"],"is_corresponding":false,"raw_author_name":"Bo An","raw_affiliation_strings":["College of Computing and Data Science, Nanyang Technological University, Jurong West, Singapore"],"raw_orcid":"https://orcid.org/0000-0002-7064-7438","affiliations":[{"raw_affiliation_string":"College of Computing and Data Science, Nanyang Technological University, Jurong West, Singapore","institution_ids":["https://openalex.org/I172675005"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029392006","display_name":"Yi Chang","orcid":"https://orcid.org/0000-0003-2697-8093"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yi Chang","raw_affiliation_strings":["Engineering Research Center of Knowledge-Driven Human-Machine Intelligence, MOE, Changchun, China"],"raw_orcid":"https://orcid.org/0000-0003-2697-8093","affiliations":[{"raw_affiliation_string":"Engineering Research Center of Knowledge-Driven Human-Machine Intelligence, MOE, Changchun, China","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.12809244,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"37","issue":"4","first_page":"1851","last_page":"1863"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9068999886512756,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9068999886512756,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.008999999612569809,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.006399999838322401,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.6378999948501587},{"id":"https://openalex.org/keywords/kullback\u2013leibler-divergence","display_name":"Kullback\u2013Leibler divergence","score":0.6044999957084656},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5927000045776367},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.5677000284194946},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.4871000051498413},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.4507000148296356},{"id":"https://openalex.org/keywords/maximum-entropy-spectral-estimation","display_name":"Maximum entropy spectral estimation","score":0.43860000371932983},{"id":"https://openalex.org/keywords/divergence","display_name":"Divergence (linguistics)","score":0.39730000495910645}],"concepts":[{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.6467999815940857},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.6378999948501587},{"id":"https://openalex.org/C171752962","wikidata":"https://www.wikidata.org/wiki/Q255166","display_name":"Kullback\u2013Leibler divergence","level":2,"score":0.6044999957084656},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5927000045776367},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.5677000284194946},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.4871000051498413},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4862000048160553},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4652999937534332},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.4507000148296356},{"id":"https://openalex.org/C89735579","wikidata":"https://www.wikidata.org/wiki/Q6795894","display_name":"Maximum entropy spectral estimation","level":3,"score":0.43860000371932983},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.4253000020980835},{"id":"https://openalex.org/C207390915","wikidata":"https://www.wikidata.org/wiki/Q1230525","display_name":"Divergence (linguistics)","level":2,"score":0.39730000495910645},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.3815999925136566},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.3431999981403351},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.326200008392334},{"id":"https://openalex.org/C126980161","wikidata":"https://www.wikidata.org/wiki/Q863783","display_name":"Simulated annealing","level":2,"score":0.30070000886917114},{"id":"https://openalex.org/C44415725","wikidata":"https://www.wikidata.org/wiki/Q4913893","display_name":"Binary entropy function","level":3,"score":0.29190000891685486},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.2793999910354614},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.2770000100135803},{"id":"https://openalex.org/C95546049","wikidata":"https://www.wikidata.org/wiki/Q1345207","display_name":"Entropy estimation","level":3,"score":0.27390000224113464},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.2676999866962433},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.26030001044273376},{"id":"https://openalex.org/C197055811","wikidata":"https://www.wikidata.org/wiki/Q207522","display_name":"Probability density function","level":2,"score":0.25699999928474426},{"id":"https://openalex.org/C41045048","wikidata":"https://www.wikidata.org/wiki/Q202843","display_name":"Linear programming","level":2,"score":0.25679999589920044},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.25619998574256897}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2025.3626050","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2025.3626050","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:41264432","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/41264432","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G1211044700","display_name":null,"funder_award_id":"U2341229","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G1212804187","display_name":null,"funder_award_id":"62476109","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6314401316","display_name":null,"funder_award_id":"202206170041","funder_id":"https://openalex.org/F4320322725","funder_display_name":"China Scholarship Council"},{"id":"https://openalex.org/G6789821807","display_name":null,"funder_award_id":"62206108","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G811175149","display_name":null,"funder_award_id":"2021ZD0112500","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8116777385","display_name":null,"funder_award_id":"62476110","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322725","display_name":"China Scholarship Council","ror":"https://ror.org/04atp4p48"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":20,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1499669280","https://openalex.org/W2052958516","https://openalex.org/W2145339207","https://openalex.org/W2158782408","https://openalex.org/W2173401535","https://openalex.org/W2746553466","https://openalex.org/W2904246096","https://openalex.org/W2981873476","https://openalex.org/W2996896271","https://openalex.org/W2998050631","https://openalex.org/W3015082424","https://openalex.org/W3088310808","https://openalex.org/W3124201714","https://openalex.org/W4214717370","https://openalex.org/W4281258657","https://openalex.org/W4319165238","https://openalex.org/W4382239271","https://openalex.org/W4396583645","https://openalex.org/W4406983280"],"related_works":[],"abstract_inverted_index":{"In":[0],"real-world":[1],"decision-making":[2],"tasks,":[3],"it":[4,42],"is":[5,158],"critical":[6],"for":[7],"reinforcement":[8],"learning":[9],"(RL)":[10],"methods":[11,21],"to":[12,88,110],"be":[13,130,142],"both":[14],"stable":[15,214],"and":[16,47,56,71,78,206,208,222,235],"robust.":[17],"Maximum":[18],"entropy":[19,28,33,85,117,147],"RL":[20],"typically":[22],"generate":[23],"a":[24,89,98,104,133,179],"robust":[25],"policy":[26,54,70,128,181,199],"with":[27,81,132],"augmented":[29],"reward.":[30],"While":[31],"incorporating":[32],"into":[34],"the":[35,38,83,115,123,127,146,149,154,163,187,197,203,220],"reward":[36],"offers":[37],"benefit":[39],"of":[40,126,162],"exploration,":[41],"presents":[43,183],"limited":[44],"universal":[45],"applicability":[46],"persistent":[48],"convergence":[49],"difficulties,":[50],"such":[51,112],"as":[52,68,153],"suboptimal":[53],"stabilization":[55],"unstable":[57],"$Q$":[58,91,139,151,188,204],"value":[59,205],"update.":[60],"From":[61],"optimization,":[62],"we":[63,169,194,209],"define":[64],"these":[65,192],"two":[66],"issues":[67,113],"tremulous":[69,99,124],"spiky":[72,90,138],"Q-function,":[73],"investigating":[74],"their":[75],"underlying":[76],"causes":[77],"relationships.":[79],"Analysis":[80],"this,":[82],"maximum":[84,116],"principle":[86],"leads":[87],"-function":[92,140],"update,":[93],"which":[94],"ultimately":[95,195],"results":[96,177,218],"in":[97,148,178,186],"policy.":[100],"We":[101],"thus":[102],"introduce":[103],"beta-symmetric":[105,155],"Kullback-Leibler":[106],"(KL)":[107],"divergence":[108,157],"objective":[109,121,175],"mitigate":[111],"under":[114],"framework.":[118],"With":[119],"this":[120,211],"function,":[122],"nature":[125],"could":[129,141],"controlled":[131],"large":[134],"beta":[135],"value.":[136,189],"The":[137],"avoided":[143],"by":[144,191,200],"annealing":[145],"target":[150],"value,":[152],"KL":[156,166],"an":[159,184],"upper":[160],"bound":[161],"original":[164],"reverse":[165],"divergence.":[167],"Theoretically,":[168],"prove":[170],"that":[171,182,226],"minimizing":[172],"our":[173,227],"new":[174,180],"function":[176],"improvement":[185],"Guaranteed":[190],"results,":[193],"derive":[196],"optimal":[198],"iteratively":[201],"updating":[202],"policy,":[207],"call":[210],"method":[212],"max-entropy":[213],"optimization":[215],"(MeSO).":[216],"Experimental":[217],"on":[219],"Mujoco":[221],"Roboschool":[223],"platforms":[224],"demonstrate":[225],"algorithm":[228],"maintains":[229],"stability":[230],"while":[231],"offering":[232],"better":[233],"flexibility":[234],"overall":[236],"performance.":[237]},"counts_by_year":[],"updated_date":"2026-07-29T09:40:50.615796","created_date":"2025-11-20T00:00:00"}