{"id":"https://openalex.org/W2965140792","doi":"https://doi.org/10.24963/ijcai.2019/475","title":"Soft Policy Gradient Method for Maximum Entropy Deep Reinforcement Learning","display_name":"Soft Policy Gradient Method for Maximum Entropy Deep Reinforcement Learning","publication_year":2019,"publication_date":"2019-07-28","ids":{"openalex":"https://openalex.org/W2965140792","doi":"https://doi.org/10.24963/ijcai.2019/475","mag":"2965140792"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2019/475","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2019/475","pdf_url":"https://www.ijcai.org/proceedings/2019/0475.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2019/0475.pdf","any_repository_has_fulltext":null},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055957441","display_name":"Wenjie Shi","orcid":"https://orcid.org/0000-0001-8677-0011"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Shi","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101868179","display_name":"Shiji Song","orcid":"https://orcid.org/0000-0003-0858-1770"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Shiji Song","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100752413","display_name":"Cheng Wu","orcid":"https://orcid.org/0000-0002-8611-2665"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cheng Wu","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5101868179"],"corresponding_institution_ids":["https://openalex.org/I99065089"],"apc_list":null,"apc_paid":null,"fwci":2.4055,"has_fulltext":false,"cited_by_count":35,"citation_normalized_percentile":{"value":0.92189285,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":null,"issue":null,"first_page":"3425","last_page":"3431"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9936000108718872,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.965399980545044,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.908600926399231},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.7534523010253906},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.7052165269851685},{"id":"https://openalex.org/keywords/curse-of-dimensionality","display_name":"Curse of dimensionality","score":0.6600918769836426},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6496313810348511},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.5156470537185669},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.5068274140357971},{"id":"https://openalex.org/keywords/soft-computing","display_name":"Soft computing","score":0.4997255802154541},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.48187336325645447},{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"Temporal difference learning","score":0.45522674918174744},{"id":"https://openalex.org/keywords/principle-of-maximum-entropy","display_name":"Principle of maximum entropy","score":0.41167575120925903},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.29671019315719604},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2293109893798828}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.908600926399231},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.7534523010253906},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.7052165269851685},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.6600918769836426},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6496313810348511},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.5156470537185669},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.5068274140357971},{"id":"https://openalex.org/C140073362","wikidata":"https://www.wikidata.org/wiki/Q738759","display_name":"Soft computing","level":3,"score":0.4997255802154541},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.48187336325645447},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.45522674918174744},{"id":"https://openalex.org/C9679016","wikidata":"https://www.wikidata.org/wiki/Q1417473","display_name":"Principle of maximum entropy","level":2,"score":0.41167575120925903},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.29671019315719604},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2293109893798828},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.24963/ijcai.2019/475","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2019/475","pdf_url":"https://www.ijcai.org/proceedings/2019/0475.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2019/475","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2019/475","pdf_url":"https://www.ijcai.org/proceedings/2019/0475.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Eighth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[{"score":0.5799999833106995,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G5006106782","display_name":null,"funder_award_id":"41427806","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2965140792.pdf","grobid_xml":"https://content.openalex.org/works/W2965140792.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W1541084404","https://openalex.org/W1771410628","https://openalex.org/W2024390895","https://openalex.org/W2098774185","https://openalex.org/W2107464055","https://openalex.org/W2119717200","https://openalex.org/W2136602922","https://openalex.org/W2145339207","https://openalex.org/W2151161180","https://openalex.org/W2155007355","https://openalex.org/W2155027007","https://openalex.org/W2155772159","https://openalex.org/W2257979135","https://openalex.org/W2554984891","https://openalex.org/W2609650878","https://openalex.org/W2754517384","https://openalex.org/W2781726626","https://openalex.org/W2949561945","https://openalex.org/W2951507724","https://openalex.org/W2962951833","https://openalex.org/W2963267001","https://openalex.org/W2963849886","https://openalex.org/W2963864421","https://openalex.org/W4302570325","https://openalex.org/W4307347247","https://openalex.org/W4400608488"],"related_works":["https://openalex.org/W3149091143","https://openalex.org/W2149418961","https://openalex.org/W2110002798","https://openalex.org/W4308702637","https://openalex.org/W4256087190","https://openalex.org/W4240668504","https://openalex.org/W2100100236","https://openalex.org/W3038962357","https://openalex.org/W2971484784","https://openalex.org/W61119710"],"abstract_inverted_index":{"Maximum":[0],"entropy":[1,76,94],"deep":[2,95,99],"reinforcement":[3],"learning":[4,116],"(RL)":[5],"methods":[6,19],"have":[7],"been":[8],"demonstrated":[9],"on":[10,27,75],"a":[11],"range":[12],"of":[13,51,121],"challenging":[14],"continuous":[15,84],"tasks.":[16],"However,":[17],"existing":[18],"either":[20],"suffer":[21],"from":[22],"severe":[23],"instability":[24],"when":[25],"training":[26],"large":[28],"off-policy":[29,90,152],"data":[30],"or":[31],"cannot":[32],"scale":[33],"to":[34,134],"tasks":[35],"with":[36,83,109],"very":[37],"high":[38],"state":[39],"and":[40],"action":[41],"dimensionality":[42],"such":[43],"as":[44],"3D":[45],"humanoid":[46],"locomotion.":[47],"Besides,":[48],"the":[49,119,136],"optimality":[50],"desired":[52],"Boltzmann":[53],"policy":[54,72,101,107],"set":[55],"for":[56,81,125],"non-optimal":[57],"soft":[58,71,100,106,110,126,137],"value":[59,127],"function":[60],"is":[61],"not":[62],"persuasive":[63],"enough.":[64],"In":[65],"this":[66],"paper,":[67],"we":[68,87,129],"first":[69],"derive":[70],"gradient":[73,102,108],"based":[74],"regularized":[77],"expected":[78],"reward":[79],"objective":[80],"RL":[82,96],"actions.":[85],"Then,":[86],"present":[88],"an":[89],"actor-critic,":[91],"model-free":[92],"maximum":[93],"algorithm":[97],"called":[98],"(DSPG)":[103],"by":[104],"combining":[105],"Bellman":[111,138],"equation.":[112],"To":[113],"ensure":[114],"stable":[115],"while":[117],"eliminating":[118],"need":[120],"two":[122],"separate":[123],"critics":[124],"functions,":[128],"leverage":[130],"double":[131],"sampling":[132],"approach":[133],"making":[135],"equation":[139],"tractable.":[140],"The":[141],"experimental":[142],"results":[143],"demonstrate":[144],"that":[145],"our":[146],"method":[147],"outperforms":[148],"in":[149],"performance":[150],"over":[151],"prior":[153],"methods.":[154]},"counts_by_year":[{"year":2025,"cited_by_count":5},{"year":2024,"cited_by_count":4},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":7},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}