{"id":"https://openalex.org/W4226493407","doi":"https://doi.org/10.1109/tnnls.2022.3203024","title":"Action Candidate Driven Clipped Double Q-Learning for Discrete and Continuous Action Tasks","display_name":"Action Candidate Driven Clipped Double Q-Learning for Discrete and Continuous Action Tasks","publication_year":2022,"publication_date":"2022-09-27","ids":{"openalex":"https://openalex.org/W4226493407","doi":"https://doi.org/10.1109/tnnls.2022.3203024","pmid":"https://pubmed.ncbi.nlm.nih.gov/36166566"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2022.3203024","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2022.3203024","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047323226","display_name":"Haobo Jiang","orcid":"https://orcid.org/0000-0002-4536-5252"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Haobo Jiang","raw_affiliation_strings":["PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0002-4536-5252","affiliations":[{"raw_affiliation_string":"PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]},{"raw_affiliation_string":"School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100412642","display_name":"Guangyu Li","orcid":"https://orcid.org/0000-0003-4817-0618"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Guangyu Li","raw_affiliation_strings":["PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-4817-0618","affiliations":[{"raw_affiliation_string":"PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]},{"raw_affiliation_string":"School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079434039","display_name":"Jin Xie","orcid":"https://orcid.org/0000-0002-9157-838X"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jin Xie","raw_affiliation_strings":["PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]},{"raw_affiliation_string":"School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100726984","display_name":"Jian Yang","orcid":"https://orcid.org/0000-0003-4800-832X"},"institutions":[{"id":"https://openalex.org/I36399199","display_name":"Nanjing University of Science and Technology","ror":"https://ror.org/00xp9wg62","country_code":"CN","type":"education","lineage":["https://openalex.org/I36399199"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Yang","raw_affiliation_strings":["PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China"],"raw_orcid":"https://orcid.org/0000-0003-4800-832X","affiliations":[{"raw_affiliation_string":"PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, School of Computer Science and Engineering, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]},{"raw_affiliation_string":"School of Computer Science and Engineering, PCA Laboratory, the Key Laboratory of Intelligent Perception and Systems for High-Dimensional Information of Ministry of Education, and the Jiangsu Key Laboratory of Image and Video Understanding for Social Security, Nanjing University of Science and Technology, Nanjing, China","institution_ids":["https://openalex.org/I36399199"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I36399199"],"apc_list":null,"apc_paid":null,"fwci":3.6081,"has_fulltext":false,"cited_by_count":28,"citation_normalized_percentile":{"value":0.93641134,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"35","issue":"4","first_page":"5269","last_page":"5279"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12535","display_name":"Machine Learning and Data Classification","score":0.9927999973297119,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10876","display_name":"Fault Detection and Control Systems","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.7715030908584595},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.5632562041282654},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.554451584815979},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.5207902789115906},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.4988577365875244},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.466068297624588},{"id":"https://openalex.org/keywords/expected-value","display_name":"Expected value","score":0.45572155714035034},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4460236132144928},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44121307134628296},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.4389110803604126},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.38302385807037354},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.35828137397766113},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.18778562545776367},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.09417426586151123}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.7715030908584595},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5632562041282654},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.554451584815979},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.5207902789115906},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4988577365875244},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.466068297624588},{"id":"https://openalex.org/C141042865","wikidata":"https://www.wikidata.org/wiki/Q200125","display_name":"Expected value","level":2,"score":0.45572155714035034},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4460236132144928},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44121307134628296},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.4389110803604126},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.38302385807037354},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.35828137397766113},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.18778562545776367},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.09417426586151123},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C13280743","wikidata":"https://www.wikidata.org/wiki/Q131089","display_name":"Geodesy","level":1,"score":0.0},{"id":"https://openalex.org/C205649164","wikidata":"https://www.wikidata.org/wiki/Q1071","display_name":"Geography","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2022.3203024","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2022.3203024","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:36166566","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/36166566","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8199999928474426,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[{"id":"https://openalex.org/G18858354","display_name":null,"funder_award_id":"BK20190444","funder_id":"https://openalex.org/F4320322769","funder_display_name":"Natural Science Foundation of Jiangsu Province"},{"id":"https://openalex.org/G5634287101","display_name":null,"funder_award_id":"61876084","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G635378778","display_name":null,"funder_award_id":"U1713208","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7456927639","display_name":null,"funder_award_id":"62006119","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320322769","display_name":"Natural Science Foundation of Jiangsu Province","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W41554520","https://openalex.org/W166862392","https://openalex.org/W1784384189","https://openalex.org/W1941445455","https://openalex.org/W2093253120","https://openalex.org/W2107726111","https://openalex.org/W2120346334","https://openalex.org/W2123447947","https://openalex.org/W2129670787","https://openalex.org/W2144744117","https://openalex.org/W2145339207","https://openalex.org/W2150468603","https://openalex.org/W2158782408","https://openalex.org/W2740912559","https://openalex.org/W2746553466","https://openalex.org/W2765302304","https://openalex.org/W2895095767","https://openalex.org/W2902098903","https://openalex.org/W2941439130","https://openalex.org/W3011120880","https://openalex.org/W3176552394","https://openalex.org/W3200276130","https://openalex.org/W3207654079","https://openalex.org/W4211221179","https://openalex.org/W6602057636","https://openalex.org/W6677067356","https://openalex.org/W6677737365","https://openalex.org/W6678481081","https://openalex.org/W6684921986","https://openalex.org/W6687590523","https://openalex.org/W6735506055","https://openalex.org/W6741821246","https://openalex.org/W6747473740","https://openalex.org/W6748839928","https://openalex.org/W6752656891","https://openalex.org/W6755475254","https://openalex.org/W6757677476","https://openalex.org/W6762358693","https://openalex.org/W6763056091","https://openalex.org/W6771936881","https://openalex.org/W6775970255","https://openalex.org/W6784003830","https://openalex.org/W6789305357","https://openalex.org/W7000276917"],"related_works":["https://openalex.org/W4400868993","https://openalex.org/W3096874164","https://openalex.org/W1985560493","https://openalex.org/W2386410636","https://openalex.org/W2357975469","https://openalex.org/W2145363145","https://openalex.org/W1626977535","https://openalex.org/W2341346307","https://openalex.org/W3168977894","https://openalex.org/W2963824430"],"abstract_inverted_index":{"Double":[0,50,79,163,196],"Q-learning":[1,51,164,197],"is":[2,147,237],"a":[3,85],"popular":[4],"reinforcement":[5],"learning":[6],"algorithm":[7,214],"in":[8,55,62,122,137,160],"Markov":[9],"decision":[10],"process":[11],"(MDP)":[12],"problems.":[13,235],"Clipped":[14],"double":[15,22,27,44,75],"Q-learning,":[16,23],"as":[17,167],"an":[18,71],"effective":[19],"variant":[20],"of":[21,41,48,87,98,114,126,133,141,170,177],"employs":[24],"the":[25,31,38,42,46,66,106,111,119,123,130,134,138,144,151,157,168,175,181,184,204,219],"clipped":[26,43,49,74,145,162,195],"estimator":[28,76],"to":[29,37,64,128,198],"approximate":[30],"maximum":[32,120,152,220],"expected":[33,153,221],"action":[34,72,89,93,109,131,136,154,171,178,200,207,222],"value.":[35,155],"Due":[36],"underestimation":[39,67,158,187],"bias":[40,159],"estimator,":[45],"performance":[47,231],"may":[52],"be":[53],"degraded":[54],"some":[56,225],"stochastic":[57],"environments.":[58],"In":[59,189],"this":[60],"article,":[61],"order":[63],"reduce":[65],"bias,":[68],"we":[69,82,104,117,191],"propose":[70],"candidate-based":[73],"(AC-CDE)":[77],"for":[78,149],"Q-learning.":[80],"Specifically,":[81],"first":[83,139],"select":[84],"set":[86,97,113,125,140],"elite":[88,205],"candidates":[90,172,179],"with":[91],"high":[92],"values":[94],"from":[95,110],"one":[96],"estimators.":[99,115],"Then,":[100],"among":[101],"these":[102],"candidates,":[103],"choose":[105],"highest":[107],"valued":[108],"other":[112],"Finally,":[116],"use":[118],"value":[121,132,146,223],"second":[124],"estimators":[127,142],"clip":[129],"chosen":[135],"and":[143,186,228],"used":[148],"approximating":[150,203],"Theoretically,":[156],"our":[161,194,213],"decays":[165],"monotonically":[166],"number":[169,176],"decreases.":[173],"Moreover,":[174],"controls":[180],"tradeoff":[182],"between":[183],"overestimation":[185],"biases.":[188],"addition,":[190],"also":[192],"extend":[193],"continuous":[199,206],"tasks":[201],"via":[202],"candidates.":[208],"We":[209],"empirically":[210],"verify":[211],"that":[212],"can":[215],"more":[216],"accurately":[217],"estimate":[218],"on":[224,232],"toy":[226],"environments":[227],"yield":[229],"good":[230],"several":[233],"benchmark":[234],"Code":[236],"available":[238],"at":[239],"https://github.com/Jiang-HB/ac_CDQ.":[240]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":14},{"year":2024,"cited_by_count":9},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":2}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}