{"id":"https://openalex.org/W3196831457","doi":"https://doi.org/10.1109/rcar52367.2021.9517647","title":"Data-efficient Deep Reinforcement Learning Method Toward Scaling Continuous Robotic Task with Sparse Rewards","display_name":"Data-efficient Deep Reinforcement Learning Method Toward Scaling Continuous Robotic Task with Sparse Rewards","publication_year":2021,"publication_date":"2021-07-15","ids":{"openalex":"https://openalex.org/W3196831457","doi":"https://doi.org/10.1109/rcar52367.2021.9517647","mag":"3196831457"},"language":"en","primary_location":{"id":"doi:10.1109/rcar52367.2021.9517647","is_oa":false,"landing_page_url":"https://doi.org/10.1109/rcar52367.2021.9517647","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE International Conference on Real-time Computing and Robotics (RCAR)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5073231890","display_name":"Junkai Ren","orcid":"https://orcid.org/0000-0002-6199-7452"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Junkai Ren","raw_affiliation_strings":["College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100616611","display_name":"Yichuan Zhang","orcid":"https://orcid.org/0000-0002-0523-3271"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yichuan Zhang","raw_affiliation_strings":["College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5003590227","display_name":"Yujun Zeng","orcid":"https://orcid.org/0000-0002-5765-684X"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yujun Zeng","raw_affiliation_strings":["College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5030762555","display_name":"Yixing Lan","orcid":"https://orcid.org/0000-0003-4503-643X"},"institutions":[{"id":"https://openalex.org/I170215575","display_name":"National University of Defense Technology","ror":"https://ror.org/05d2yfz11","country_code":"CN","type":"education","lineage":["https://openalex.org/I170215575"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yixing Lan","raw_affiliation_strings":["College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"College of Intelligence Science and Technology, National University of Defense Technology, Changsha, China","institution_ids":["https://openalex.org/I170215575"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I170215575"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.10860054,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"1425","last_page":"1431"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9919999837875366,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9682000279426575,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8677352070808411},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7562962770462036},{"id":"https://openalex.org/keywords/hindsight-bias","display_name":"Hindsight bias","score":0.6983041763305664},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6789615154266357},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6270464062690735},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5632374286651611},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4866686463356018},{"id":"https://openalex.org/keywords/property","display_name":"Property (philosophy)","score":0.44733697175979614},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4341452121734619},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08443492650985718}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8677352070808411},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7562962770462036},{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.6983041763305664},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6789615154266357},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6270464062690735},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5632374286651611},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4866686463356018},{"id":"https://openalex.org/C189950617","wikidata":"https://www.wikidata.org/wiki/Q937228","display_name":"Property (philosophy)","level":2,"score":0.44733697175979614},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4341452121734619},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08443492650985718},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/rcar52367.2021.9517647","is_oa":false,"landing_page_url":"https://doi.org/10.1109/rcar52367.2021.9517647","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 IEEE International Conference on Real-time Computing and Robotics (RCAR)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1757796397","https://openalex.org/W2082511574","https://openalex.org/W2116386744","https://openalex.org/W2257979135","https://openalex.org/W2296073425","https://openalex.org/W2347074400","https://openalex.org/W2575705757","https://openalex.org/W2738318237","https://openalex.org/W2772709170","https://openalex.org/W2781726626","https://openalex.org/W2787938642","https://openalex.org/W2789008106","https://openalex.org/W2954718200","https://openalex.org/W2960876848","https://openalex.org/W2962902376","https://openalex.org/W2963311874","https://openalex.org/W2963401755","https://openalex.org/W2963508354","https://openalex.org/W2963669336","https://openalex.org/W2963864421","https://openalex.org/W2963923407","https://openalex.org/W2964001908","https://openalex.org/W2964043796","https://openalex.org/W2964120017","https://openalex.org/W2996037775","https://openalex.org/W3005971017","https://openalex.org/W3007769740","https://openalex.org/W3122688976","https://openalex.org/W3156829097","https://openalex.org/W4285620206","https://openalex.org/W4298857966","https://openalex.org/W4300799055","https://openalex.org/W6637967152","https://openalex.org/W6671000503","https://openalex.org/W6684921986","https://openalex.org/W6692846177","https://openalex.org/W6740801417","https://openalex.org/W6741302124","https://openalex.org/W6745347688","https://openalex.org/W6746177919","https://openalex.org/W6747473740","https://openalex.org/W6748839928","https://openalex.org/W6759312711","https://openalex.org/W6772005887","https://openalex.org/W6780559895","https://openalex.org/W6783630529"],"related_works":["https://openalex.org/W3197854638","https://openalex.org/W3140454661","https://openalex.org/W4245029315","https://openalex.org/W2139970489","https://openalex.org/W1492315459","https://openalex.org/W1512434910","https://openalex.org/W2540910169","https://openalex.org/W3012552522","https://openalex.org/W2110944602","https://openalex.org/W3213722473"],"abstract_inverted_index":{"Dealing":[0],"with":[1,7,52,112,232],"the":[2,57,64,91,105,142,146,155,160,167,195,206,213,221],"robotic":[3,47,110,187,217],"continuous":[4,109],"control":[5],"problem":[6],"sparse":[8,53,113,233],"rewards":[9],"is":[10,76,144,163,182,224],"a":[11,77,96,185,190,199,237],"longstanding":[12],"challenge":[13],"in":[14,27],"deep":[15],"reinforcement":[16],"learning":[17,28,33,65,145,239],"(RL).":[18],"While":[19],"existing":[20],"DRL":[21,68,201],"algorithms":[22],"have":[23],"demonstrated":[24],"great":[25],"progress":[26],"policies":[29,35],"from":[30],"visual":[31],"observations,":[32],"effective":[34],"still":[36],"requires":[37],"an":[38],"impractical":[39],"number":[40],"of":[41,159,180,208,212,226],"real-world":[42],"data":[43,59],"samples.":[44],"Moreover,":[45],"some":[46],"tasks":[48,111],"are":[49],"naturally":[50],"specified":[51],"rewards,":[54,234],"which":[55,162],"makes":[56],"precious":[58],"inefficient":[60],"and":[61,86,128,169,203,235],"slows":[62],"down":[63],"process,":[66],"making":[67],"infeasible.":[69],"In":[70],"addition,":[71],"manually":[72],"shaping":[73],"reward":[74],"functions":[75],"complex":[78],"work":[79],"because":[80],"it":[81,152,170],"needs":[82],"specific":[83],"domain":[84],"knowledge":[85],"human":[87],"intervention.":[88],"To":[89,115],"alleviate":[90],"issue,":[92],"this":[93],"paper":[94],"proposes":[95],"model-free,":[97],"off-policy":[98],"RL":[99],"approach":[100,223],"named":[101],"TD3MHER,":[102],"to":[103,134,153,165,193,198,204],"learn":[104,154],"manipulating":[106],"policy":[107,124],"for":[108],"rewards.":[114],"be":[116],"specific,":[117],"TD3MHER":[118,148,181],"utilizes":[119],"Twin":[120],"Delayed":[121],"Deep":[122],"Deterministic":[123],"gradient":[125],"algorithm":[126,202],"(TD3)":[127],"Model-driven":[129],"Hindsight":[130],"Experience":[131],"Replay":[132],"(MHER)":[133],"achieve":[135],"highly":[136],"sample-efficient":[137],"training":[138],"property.":[139],"Because":[140],"while":[141],"agent":[143],"policy,":[147],"could":[149],"also":[150],"help":[151],"potation":[156],"physical":[157],"model":[158],"robot":[161],"helpful":[164],"solve":[166],"task,":[168],"does":[171],"not":[172],"necessitate":[173],"any":[174],"novel":[175],"robot-environment":[176],"interactions.":[177],"The":[178],"performance":[179],"assessed":[183],"on":[184,215],"simulated":[186,216],"task":[188,218],"using":[189],"7-DOF":[191],"manipulator":[192],"compare":[194],"proposed":[196,222],"technique":[197],"previous":[200],"verify":[205],"usefulness":[207],"our":[209],"method.":[210],"Results":[211],"experiments":[214],"show":[219],"that":[220],"capable":[225],"successfully":[227],"utilizing":[228],"previously":[229],"store":[230],"samples":[231],"obtain":[236],"faster":[238],"speed.":[240]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-29T14:22:42.915294","created_date":"2025-10-10T00:00:00"}