{"id":"https://openalex.org/W2907217398","doi":"https://doi.org/10.1109/tnnls.2018.2884797","title":"Multi Pseudo Q-Learning-Based Deterministic Policy Gradient for Tracking Control of Autonomous Underwater Vehicles","display_name":"Multi Pseudo Q-Learning-Based Deterministic Policy Gradient for Tracking Control of Autonomous Underwater Vehicles","publication_year":2018,"publication_date":"2018-12-28","ids":{"openalex":"https://openalex.org/W2907217398","doi":"https://doi.org/10.1109/tnnls.2018.2884797","mag":"2907217398","pmid":"https://pubmed.ncbi.nlm.nih.gov/30602426"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2018.2884797","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2018.2884797","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5055957441","display_name":"Wenjie Shi","orcid":"https://orcid.org/0000-0001-8677-0011"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Shi","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8677-0011","affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101868179","display_name":"Shiji Song","orcid":"https://orcid.org/0000-0003-0858-1770"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shiji Song","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-0858-1770","affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100752413","display_name":"Cheng Wu","orcid":"https://orcid.org/0000-0002-8611-2665"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Cheng Wu","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8611-2665","affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100643265","display_name":"C. L. Philip Chen","orcid":"https://orcid.org/0000-0001-5451-7230"},"institutions":[{"id":"https://openalex.org/I204512498","display_name":"University of Macau","ror":"https://ror.org/01r4q9n85","country_code":"MO","type":"education","lineage":["https://openalex.org/I204512498"]}],"countries":["MO"],"is_corresponding":false,"raw_author_name":"C. L. Philip Chen","raw_affiliation_strings":["Department of Computer and Information Science, Faculty of Science and Technology, University of Macau, Macau, China"],"raw_orcid":"https://orcid.org/0000-0001-5451-7230","affiliations":[{"raw_affiliation_string":"Department of Computer and Information Science, Faculty of Science and Technology, University of Macau, Macau, China","institution_ids":["https://openalex.org/I204512498"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":7.2157,"has_fulltext":false,"cited_by_count":113,"citation_normalized_percentile":{"value":0.97687456,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":"30","issue":"12","first_page":"3534","last_page":"3546"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9987000226974487,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10675","display_name":"Mechanical Circulatory Support Devices","score":0.9413999915122986,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6037588715553284},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5869936347007751},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.5798295736312866},{"id":"https://openalex.org/keywords/generality","display_name":"Generality","score":0.5612253546714783},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5459004640579224},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5320694446563721},{"id":"https://openalex.org/keywords/tracking-error","display_name":"Tracking error","score":0.510941207408905},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.49860525131225586},{"id":"https://openalex.org/keywords/tracking","display_name":"Tracking (education)","score":0.49524274468421936},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.4945010840892792},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.48560065031051636},{"id":"https://openalex.org/keywords/underactuation","display_name":"Underactuation","score":0.4646017253398895},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4612371623516083},{"id":"https://openalex.org/keywords/control-theory","display_name":"Control theory (sociology)","score":0.4530564844608307},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.44475504755973816},{"id":"https://openalex.org/keywords/value","display_name":"Value (mathematics)","score":0.43097907304763794},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.381350040435791},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.2823418974876404},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.2621725797653198},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.22089603543281555},{"id":"https://openalex.org/keywords/economics","display_name":"Economics","score":0.09073948860168457}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6037588715553284},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5869936347007751},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5798295736312866},{"id":"https://openalex.org/C2780767217","wikidata":"https://www.wikidata.org/wiki/Q5532421","display_name":"Generality","level":2,"score":0.5612253546714783},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5459004640579224},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5320694446563721},{"id":"https://openalex.org/C183356978","wikidata":"https://www.wikidata.org/wiki/Q1779213","display_name":"Tracking error","level":3,"score":0.510941207408905},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.49860525131225586},{"id":"https://openalex.org/C2775936607","wikidata":"https://www.wikidata.org/wiki/Q466845","display_name":"Tracking (education)","level":2,"score":0.49524274468421936},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4945010840892792},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.48560065031051636},{"id":"https://openalex.org/C88337583","wikidata":"https://www.wikidata.org/wiki/Q7883433","display_name":"Underactuation","level":3,"score":0.4646017253398895},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4612371623516083},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.4530564844608307},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.44475504755973816},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.43097907304763794},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.381350040435791},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2823418974876404},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2621725797653198},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.22089603543281555},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.09073948860168457},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2018.2884797","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tnnls.2018.2884797","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:30602426","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/30602426","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Life below water","id":"https://metadata.un.org/sdg/14","score":0.5400000214576721}],"awards":[{"id":"https://openalex.org/G5006106782","display_name":null,"funder_award_id":"41427806","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W158678736","https://openalex.org/W172062838","https://openalex.org/W1544853044","https://openalex.org/W1811949033","https://openalex.org/W1977671496","https://openalex.org/W1986024064","https://openalex.org/W1996099152","https://openalex.org/W1997880753","https://openalex.org/W2017957151","https://openalex.org/W2036316876","https://openalex.org/W2082526944","https://openalex.org/W2087963019","https://openalex.org/W2089916727","https://openalex.org/W2105551637","https://openalex.org/W2121863487","https://openalex.org/W2122075174","https://openalex.org/W2135783539","https://openalex.org/W2142620596","https://openalex.org/W2145339207","https://openalex.org/W2153596355","https://openalex.org/W2155968351","https://openalex.org/W2156387975","https://openalex.org/W2164313740","https://openalex.org/W2165150801","https://openalex.org/W2173248099","https://openalex.org/W2253274025","https://openalex.org/W2289659279","https://openalex.org/W2292610103","https://openalex.org/W2343912161","https://openalex.org/W2345111710","https://openalex.org/W2372306012","https://openalex.org/W2388591084","https://openalex.org/W2390714945","https://openalex.org/W2411333092","https://openalex.org/W2472686038","https://openalex.org/W2492811885","https://openalex.org/W2549225575","https://openalex.org/W2583993537","https://openalex.org/W2586823359","https://openalex.org/W2590087162","https://openalex.org/W2592873849","https://openalex.org/W2612054959","https://openalex.org/W2733312032","https://openalex.org/W2742454413","https://openalex.org/W2746553466","https://openalex.org/W2963428623","https://openalex.org/W2963864421","https://openalex.org/W2964121744","https://openalex.org/W3026829485","https://openalex.org/W3100944043","https://openalex.org/W3106462682","https://openalex.org/W4302570325","https://openalex.org/W6607059912","https://openalex.org/W6682889407","https://openalex.org/W6684205842","https://openalex.org/W6696156361","https://openalex.org/W6742283387"],"related_works":["https://openalex.org/W2045049461","https://openalex.org/W1978893398","https://openalex.org/W2201908702","https://openalex.org/W4381094582","https://openalex.org/W2369625323","https://openalex.org/W2364579609","https://openalex.org/W1977906818","https://openalex.org/W1522139108","https://openalex.org/W2353528968","https://openalex.org/W3163501229"],"abstract_inverted_index":{"This":[0],"paper":[1],"investigates":[2],"trajectory":[3],"tracking":[4,36,48,229],"problem":[5],"for":[6,79,114,133,157],"a":[7,58,71],"class":[8],"of":[9,51,148,179,192,202,235,246],"underactuated":[10],"autonomous":[11],"underwater":[12],"vehicles":[13],"(AUVs)":[14],"with":[15,109,218],"unknown":[16],"dynamics":[17],"and":[18,39,53,65,74,137,151,169,200,232,249],"constrained":[19],"inputs.":[20],"Different":[21],"from":[22],"existing":[23],"policy":[24,73,123,128,161,173,207],"gradient":[25,162,208],"methods":[26],"which":[27,120],"employ":[28],"single":[29],"actor":[30],"critic":[31,95],"but":[32,185],"cannot":[33],"realize":[34],"satisfactory":[35],"control":[37,49,230],"accuracy":[38,50,231],"stable":[40,54,233],"learning,":[41],"our":[42],"proposed":[43,143,204],"algorithm":[44,210],"can":[45],"achieve":[46],"high-level":[47,228],"AUVs":[52],"learning":[55,194,234],"by":[56,213],"applying":[57],"hybrid":[59],"actors-critics":[60],"architecture,":[61],"where":[62],"multiple":[63],"actors":[64,181,248],"critics":[66,250],"are":[67,211],"trained":[68],"to":[69,91,96,104,124,144,152,165,182],"learn":[70],"deterministic":[72,160,206],"action-value":[75,149],"function,":[76],"respectively.":[77],"Specifically,":[78],"the":[80,82,93,106,115,126,146,154,158,167,170,177,189,193,203,214,225,238,244,247,254],"critics,":[81],"expected":[83],"absolute":[84],"Bellman":[85],"error-based":[86],"updating":[87],"rule":[88],"is":[89,131,142,163,174,195],"used":[90],"choose":[92],"worst":[94],"be":[97],"updated":[98],"in":[99,129],"each":[100],"time":[101],"step.":[102],"Subsequently,":[103],"calculate":[105],"loss":[107],"function":[108,150],"more":[110],"accurate":[111],"target":[112],"value":[113],"chosen":[116],"critic,":[117],"Pseudo":[118,139],"Q-learning,":[119,130],"uses":[121],"subgreedy":[122],"replace":[125],"greedy":[127],"developed":[132],"continuous":[134],"action":[135],"spaces,":[136],"Multi":[138],"Q-learning":[140],"(MPQ)":[141],"reduce":[145],"overestimation":[147],"stabilize":[153],"learning.":[155],"As":[156],"actors,":[159],"applied":[164],"update":[166],"weights,":[168],"final":[171],"learned":[172],"defined":[175],"as":[176],"average":[178],"all":[180],"avoid":[183],"large":[184],"bad":[186],"updates.":[187],"Moreover,":[188],"stability":[190],"analysis":[191],"given":[196],"qualitatively.":[197],"The":[198],"effectiveness":[199],"generality":[201],"MPQ-based":[205],"(MPQ-DPG)":[209],"verified":[212],"application":[215],"on":[216],"AUV":[217],"two":[219],"different":[220],"reference":[221],"trajectories.":[222],"In":[223],"addition,":[224],"results":[226,239],"demonstrate":[227],"MPQ-DPG.":[236],"Besides,":[237],"also":[240],"validate":[241],"that":[242],"increasing":[243],"number":[245],"will":[251],"further":[252],"improve":[253],"performance.":[255]},"counts_by_year":[{"year":2026,"cited_by_count":4},{"year":2025,"cited_by_count":13},{"year":2024,"cited_by_count":14},{"year":2023,"cited_by_count":25},{"year":2022,"cited_by_count":19},{"year":2021,"cited_by_count":21},{"year":2020,"cited_by_count":14},{"year":2019,"cited_by_count":3}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}