{"id":"https://openalex.org/W4390938908","doi":"https://doi.org/10.1109/tai.2024.3354694","title":"Proximal Policy Optimization With Advantage Reuse Competition","display_name":"Proximal Policy Optimization With Advantage Reuse Competition","publication_year":2024,"publication_date":"2024-01-17","ids":{"openalex":"https://openalex.org/W4390938908","doi":"https://doi.org/10.1109/tai.2024.3354694"},"language":"en","primary_location":{"id":"doi:10.1109/tai.2024.3354694","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2024.3354694","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5091364297","display_name":"Yuhu Cheng","orcid":"https://orcid.org/0000-0003-2022-9999"},"institutions":[{"id":"https://openalex.org/I1327237609","display_name":"Ministry of Education of the People's Republic of China","ror":"https://ror.org/01mv9t934","country_code":"CN","type":"government","lineage":["https://openalex.org/I1327237609","https://openalex.org/I4210127390"]},{"id":"https://openalex.org/I25757504","display_name":"China University of Mining and Technology","ror":"https://ror.org/01xt2dr21","country_code":"CN","type":"education","lineage":["https://openalex.org/I25757504"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuhu Cheng","raw_affiliation_strings":["Engineering Research Center of Intelligent Control for Underground Space, Ministry of Education, Xuzhou, China","School of Information and Control Engineering, China University of Mining and Technology, Xuzhou, China"],"raw_orcid":"https://orcid.org/0000-0003-2022-9999","affiliations":[{"raw_affiliation_string":"Engineering Research Center of Intelligent Control for Underground Space, Ministry of Education, Xuzhou, China","institution_ids":["https://openalex.org/I1327237609"]},{"raw_affiliation_string":"School of Information and Control Engineering, China University of Mining and Technology, Xuzhou, China","institution_ids":["https://openalex.org/I25757504"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5107026383","display_name":"Qingbang Guo","orcid":"https://orcid.org/0009-0001-1847-869X"},"institutions":[{"id":"https://openalex.org/I1327237609","display_name":"Ministry of Education of the People's Republic of China","ror":"https://ror.org/01mv9t934","country_code":"CN","type":"government","lineage":["https://openalex.org/I1327237609","https://openalex.org/I4210127390"]},{"id":"https://openalex.org/I25757504","display_name":"China University of Mining and Technology","ror":"https://ror.org/01xt2dr21","country_code":"CN","type":"education","lineage":["https://openalex.org/I25757504"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Qingbang Guo","raw_affiliation_strings":["Engineering Research Center of Intelligent Control for Underground Space, Ministry of Education, Xuzhou, China","School of Information and Control Engineering, China University of Mining and Technology, Xuzhou, China"],"raw_orcid":"https://orcid.org/0009-0001-1847-869X","affiliations":[{"raw_affiliation_string":"Engineering Research Center of Intelligent Control for Underground Space, Ministry of Education, Xuzhou, China","institution_ids":["https://openalex.org/I1327237609"]},{"raw_affiliation_string":"School of Information and Control Engineering, China University of Mining and Technology, Xuzhou, China","institution_ids":["https://openalex.org/I25757504"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5108064895","display_name":"Xuesong Wang","orcid":"https://orcid.org/0000-0002-5327-1088"},"institutions":[{"id":"https://openalex.org/I1327237609","display_name":"Ministry of Education of the People's Republic of China","ror":"https://ror.org/01mv9t934","country_code":"CN","type":"government","lineage":["https://openalex.org/I1327237609","https://openalex.org/I4210127390"]},{"id":"https://openalex.org/I25757504","display_name":"China University of Mining and Technology","ror":"https://ror.org/01xt2dr21","country_code":"CN","type":"education","lineage":["https://openalex.org/I25757504"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Xuesong Wang","raw_affiliation_strings":["Engineering Research Center of Intelligent Control for Underground Space, Ministry of Education, Xuzhou, China","School of Information and Control Engineering, China University of Mining and Technology, Xuzhou, China"],"raw_orcid":"https://orcid.org/0000-0002-5327-1088","affiliations":[{"raw_affiliation_string":"Engineering Research Center of Intelligent Control for Underground Space, Ministry of Education, Xuzhou, China","institution_ids":["https://openalex.org/I1327237609"]},{"raw_affiliation_string":"School of Information and Control Engineering, China University of Mining and Technology, Xuzhou, China","institution_ids":["https://openalex.org/I25757504"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.4431,"has_fulltext":false,"cited_by_count":14,"citation_normalized_percentile":{"value":0.92809392,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"5","issue":"8","first_page":"3915","last_page":"3925"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9309999942779541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9309999942779541,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12676","display_name":"Machine Learning and ELM","score":0.9257000088691711,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6759305000305176},{"id":"https://openalex.org/keywords/inefficiency","display_name":"Inefficiency","score":0.5326071977615356},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5165785551071167},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4854767620563507},{"id":"https://openalex.org/keywords/reuse","display_name":"Reuse","score":0.4822255074977875},{"id":"https://openalex.org/keywords/competition","display_name":"Competition (biology)","score":0.47110989689826965},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.42439383268356323},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.2205927073955536},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.17695394158363342},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.1686168611049652},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.15711989998817444},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14910423755645752},{"id":"https://openalex.org/keywords/microeconomics","display_name":"Microeconomics","score":0.11702102422714233}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6759305000305176},{"id":"https://openalex.org/C2778869765","wikidata":"https://www.wikidata.org/wiki/Q6028363","display_name":"Inefficiency","level":2,"score":0.5326071977615356},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5165785551071167},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4854767620563507},{"id":"https://openalex.org/C206588197","wikidata":"https://www.wikidata.org/wiki/Q846574","display_name":"Reuse","level":2,"score":0.4822255074977875},{"id":"https://openalex.org/C91306197","wikidata":"https://www.wikidata.org/wiki/Q45767","display_name":"Competition (biology)","level":2,"score":0.47110989689826965},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.42439383268356323},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.2205927073955536},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.17695394158363342},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.1686168611049652},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.15711989998817444},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14910423755645752},{"id":"https://openalex.org/C175444787","wikidata":"https://www.wikidata.org/wiki/Q39072","display_name":"Microeconomics","level":1,"score":0.11702102422714233},{"id":"https://openalex.org/C548081761","wikidata":"https://www.wikidata.org/wiki/Q180388","display_name":"Waste management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tai.2024.3354694","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2024.3354694","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/1","score":0.46000000834465027,"display_name":"No poverty"}],"awards":[{"id":"https://openalex.org/G4756161396","display_name":null,"funder_award_id":"62176259","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G4823360002","display_name":null,"funder_award_id":"62373364","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W2139053308","https://openalex.org/W2201912979","https://openalex.org/W2736601468","https://openalex.org/W2951054802","https://openalex.org/W2951278471","https://openalex.org/W3049574859","https://openalex.org/W3099303207","https://openalex.org/W3135203571","https://openalex.org/W3188220908","https://openalex.org/W3194724567","https://openalex.org/W3200885897","https://openalex.org/W3206369550","https://openalex.org/W4200129022","https://openalex.org/W4214717370","https://openalex.org/W4285216091","https://openalex.org/W4293428470","https://openalex.org/W4312309366","https://openalex.org/W4312453724","https://openalex.org/W4316021747","https://openalex.org/W4317038451","https://openalex.org/W4386524114","https://openalex.org/W6638018090","https://openalex.org/W6677916085","https://openalex.org/W6679257226","https://openalex.org/W6737893269","https://openalex.org/W6741002519","https://openalex.org/W6803550857"],"related_works":["https://openalex.org/W2264067234","https://openalex.org/W3124243301","https://openalex.org/W1571502335","https://openalex.org/W1589409554","https://openalex.org/W2759038785","https://openalex.org/W2172232600","https://openalex.org/W3123876860","https://openalex.org/W3124172198","https://openalex.org/W2142633247","https://openalex.org/W2148394657"],"abstract_inverted_index":{"In":[0],"recent":[1],"years,":[2],"reinforcement":[3],"learning":[4],"(RL)":[5],"has":[6],"made":[7],"great":[8],"achievements":[9],"in":[10],"artificial":[11],"intelligence.":[12],"Proximal":[13],"policy":[14,28,33,47,72,87,91,105,129,133,146,151],"optimization":[15,73,92,134],"(PPO)":[16],"is":[17,56,78,98,115,139],"a":[18],"representative":[19],"RL":[20],"algorithm,":[21,39],"which":[22,58,100,141],"limits":[23,144],"the":[24,53,60,64,70,83,89,104,109,119,126,131,143,150,161],"magnitude":[25],"of":[26,66,86,121,128,145,163],"each":[27],"update":[29,147],"to":[30,81,117,124],"achieve":[31],"monotonic":[32],"improvement.":[34],"However,":[35],"as":[36],"an":[37],"on-policy":[38],"PPO":[40],"suffers":[41],"from":[42],"sample":[43],"inefficiency":[44],"and":[45,69,107,113,167],"poor":[46],"exploratory.":[48],"To":[49],"solve":[50],"above":[51],"problems,":[52],"off-policy":[54],"advantage":[55,61,75,94],"proposed,":[57,99,140],"calculates":[59],"function":[62],"through":[63],"reuse":[65,76,95],"previous":[67],"policy,":[68],"proximal":[71,90,132],"with":[74,93,135],"(PPO-AR)":[77],"proposed.":[79],"Furthermore,":[80],"improve":[82,118,125],"sampling":[84],"efficiency":[85],"update,":[88,130],"competition":[96],"(PPO-ARC)":[97],"introduces":[101],"PPO-AR":[102],"into":[103],"calculation":[106],"uses":[108],"parallel":[110],"competitive":[111],"optimization,":[112],"it":[114],"shown":[116],"performance":[120],"policy.":[122],"Moreover,":[123],"exploratory":[127],"generalized":[136],"clipping":[137,153],"(PPO-GC)":[138],"relaxes":[142],"by":[148],"changing":[149],"flat":[152],"boundary.":[154],"Experimental":[155],"results":[156],"on":[157],"OpenAI":[158],"Gym":[159],"demonstrate":[160],"effectiveness":[162],"our":[164],"proposed":[165],"PPO-ARC":[166],"PPO-GC.":[168]},"counts_by_year":[{"year":2026,"cited_by_count":5},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2025-10-10T00:00:00"}