{"id":"https://openalex.org/W4324116526","doi":"https://doi.org/10.1109/tase.2023.3254583","title":"Proximal Policy Optimization With Time-Varying Muscle Synergy for the Control of an Upper Limb Musculoskeletal System","display_name":"Proximal Policy Optimization With Time-Varying Muscle Synergy for the Control of an Upper Limb Musculoskeletal System","publication_year":2023,"publication_date":"2023-03-13","ids":{"openalex":"https://openalex.org/W4324116526","doi":"https://doi.org/10.1109/tase.2023.3254583"},"language":"en","primary_location":{"id":"doi:10.1109/tase.2023.3254583","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tase.2023.3254583","pdf_url":null,"source":{"id":"https://openalex.org/S34881539","display_name":"IEEE Transactions on Automation Science and Engineering","issn_l":"1545-5955","issn":["1545-5955","1558-3783"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automation Science and Engineering","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5034909379","display_name":"Rong Liu","orcid":"https://orcid.org/0000-0002-3238-5705"},"institutions":[{"id":"https://openalex.org/I27357992","display_name":"Dalian University of Technology","ror":"https://ror.org/023hj5876","country_code":"CN","type":"education","lineage":["https://openalex.org/I27357992"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Rong Liu","raw_affiliation_strings":["Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China"],"raw_orcid":"https://orcid.org/0000-0002-3238-5705","affiliations":[{"raw_affiliation_string":"Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China","institution_ids":["https://openalex.org/I27357992"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100717652","display_name":"Jiaxing Wang","orcid":"https://orcid.org/0000-0002-6061-3445"},"institutions":[{"id":"https://openalex.org/I27357992","display_name":"Dalian University of Technology","ror":"https://ror.org/023hj5876","country_code":"CN","type":"education","lineage":["https://openalex.org/I27357992"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jiaxing Wang","raw_affiliation_strings":["Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China","institution_ids":["https://openalex.org/I27357992"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5062177750","display_name":"Yaru Chen","orcid":"https://orcid.org/0000-0002-7585-0590"},"institutions":[{"id":"https://openalex.org/I27357992","display_name":"Dalian University of Technology","ror":"https://ror.org/023hj5876","country_code":"CN","type":"education","lineage":["https://openalex.org/I27357992"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaru Chen","raw_affiliation_strings":["Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China"],"raw_orcid":"https://orcid.org/0000-0002-7585-0590","affiliations":[{"raw_affiliation_string":"Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China","institution_ids":["https://openalex.org/I27357992"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100411662","display_name":"Yin Liu","orcid":"https://orcid.org/0000-0002-3677-2278"},"institutions":[{"id":"https://openalex.org/I27357992","display_name":"Dalian University of Technology","ror":"https://ror.org/023hj5876","country_code":"CN","type":"education","lineage":["https://openalex.org/I27357992"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yin Liu","raw_affiliation_strings":["Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China"],"raw_orcid":"https://orcid.org/0000-0002-3677-2278","affiliations":[{"raw_affiliation_string":"Liaoning Key Laboratory of IC &#x0026; BME System, Faculty of Electronic Information and Electrical Engineering, School of Biomedical Engineering, Dalian University of Technology, Dalian, China","institution_ids":["https://openalex.org/I27357992"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074850927","display_name":"Yongxuan Wang","orcid":"https://orcid.org/0000-0002-4753-6545"},"institutions":[{"id":"https://openalex.org/I4210092944","display_name":"Dalian University","ror":"https://ror.org/00g2ypp58","country_code":"CN","type":"education","lineage":["https://openalex.org/I4210092944"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yongxuan Wang","raw_affiliation_strings":["School of Information Engineering, Dalian University, Dalian, China"],"raw_orcid":"https://orcid.org/0000-0002-4753-6545","affiliations":[{"raw_affiliation_string":"School of Information Engineering, Dalian University, Dalian, China","institution_ids":["https://openalex.org/I4210092944"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5010340252","display_name":"Jason Gu","orcid":"https://orcid.org/0000-0002-7626-1077"},"institutions":[{"id":"https://openalex.org/I129902397","display_name":"Dalhousie University","ror":"https://ror.org/01e6qks80","country_code":"CA","type":"education","lineage":["https://openalex.org/I129902397"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jason Gu","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Dalhousie University, Halifax, NS, Canada"],"raw_orcid":"https://orcid.org/0000-0002-7626-1077","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Dalhousie University, Halifax, NS, Canada","institution_ids":["https://openalex.org/I129902397"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":6,"corresponding_author_ids":["https://openalex.org/A5034909379"],"corresponding_institution_ids":["https://openalex.org/I27357992"],"apc_list":null,"apc_paid":null,"fwci":2.2534,"has_fulltext":false,"cited_by_count":21,"citation_normalized_percentile":{"value":0.8746076,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":99},"biblio":{"volume":"21","issue":"2","first_page":"1929","last_page":"1940"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10784","display_name":"Muscle activation and electromyography studies","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10784","display_name":"Muscle activation and electromyography studies","score":1.0,"subfield":{"id":"https://openalex.org/subfields/2204","display_name":"Biomedical Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10982","display_name":"Motor Control and Adaptation","score":0.9991000294685364,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9980000257492065,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.5960793495178223},{"id":"https://openalex.org/keywords/flexibility","display_name":"Flexibility (engineering)","score":0.5558282136917114},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5351541638374329},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.5140618085861206},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5116263031959534},{"id":"https://openalex.org/keywords/adaptability","display_name":"Adaptability","score":0.5034341216087341},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.4903841018676758},{"id":"https://openalex.org/keywords/motion","display_name":"Motion (physics)","score":0.45244497060775757},{"id":"https://openalex.org/keywords/control-system","display_name":"Control system","score":0.4196721017360687},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.22088178992271423},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.1982259750366211},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.1493985950946808}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.5960793495178223},{"id":"https://openalex.org/C2780598303","wikidata":"https://www.wikidata.org/wiki/Q65921492","display_name":"Flexibility (engineering)","level":2,"score":0.5558282136917114},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5351541638374329},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5140618085861206},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5116263031959534},{"id":"https://openalex.org/C177606310","wikidata":"https://www.wikidata.org/wiki/Q5674297","display_name":"Adaptability","level":2,"score":0.5034341216087341},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.4903841018676758},{"id":"https://openalex.org/C104114177","wikidata":"https://www.wikidata.org/wiki/Q79782","display_name":"Motion (physics)","level":2,"score":0.45244497060775757},{"id":"https://openalex.org/C17500928","wikidata":"https://www.wikidata.org/wiki/Q959968","display_name":"Control system","level":2,"score":0.4196721017360687},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.22088178992271423},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.1982259750366211},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.1493985950946808},{"id":"https://openalex.org/C104317684","wikidata":"https://www.wikidata.org/wiki/Q7187","display_name":"Gene","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.0},{"id":"https://openalex.org/C55493867","wikidata":"https://www.wikidata.org/wiki/Q7094","display_name":"Biochemistry","level":1,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tase.2023.3254583","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tase.2023.3254583","pdf_url":null,"source":{"id":"https://openalex.org/S34881539","display_name":"IEEE Transactions on Automation Science and Engineering","issn_l":"1545-5955","issn":["1545-5955","1558-3783"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automation Science and Engineering","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.9100000262260437,"display_name":"Affordable and clean energy","id":"https://metadata.un.org/sdg/7"}],"awards":[{"id":"https://openalex.org/G1496771174","display_name":null,"funder_award_id":"DUT21JC19","funder_id":"https://openalex.org/F4320335787","funder_display_name":"Fundamental Research Funds for the Central Universities"},{"id":"https://openalex.org/G2548045343","display_name":null,"funder_award_id":"2020-KF-12-04","funder_id":"https://openalex.org/F4320326907","funder_display_name":"State Key Laboratory of Robotics"},{"id":"https://openalex.org/G3110425489","display_name":null,"funder_award_id":"jyt-dldxfw202005","funder_id":"https://openalex.org/F4320327473","funder_display_name":"Department of Science and Technology of Liaoning Province"},{"id":"https://openalex.org/G8653402302","display_name":null,"funder_award_id":"2022YFC2405500","funder_id":"https://openalex.org/F4320335777","funder_display_name":"National Key Research and Development Program of China"}],"funders":[{"id":"https://openalex.org/F4320326907","display_name":"State Key Laboratory of Robotics","ror":null},{"id":"https://openalex.org/F4320327473","display_name":"Department of Science and Technology of Liaoning Province","ror":"https://ror.org/02f09cx53"},{"id":"https://openalex.org/F4320335777","display_name":"National Key Research and Development Program of China","ror":null},{"id":"https://openalex.org/F4320335787","display_name":"Fundamental Research Funds for the Central Universities","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":35,"referenced_works":["https://openalex.org/W1779938893","https://openalex.org/W1972837991","https://openalex.org/W1998152406","https://openalex.org/W2028031284","https://openalex.org/W2036826589","https://openalex.org/W2058327271","https://openalex.org/W2067690526","https://openalex.org/W2070510586","https://openalex.org/W2075861930","https://openalex.org/W2103086961","https://openalex.org/W2112802476","https://openalex.org/W2130281033","https://openalex.org/W2141943257","https://openalex.org/W2151906934","https://openalex.org/W2157780858","https://openalex.org/W2276477818","https://openalex.org/W2398264187","https://openalex.org/W2519125098","https://openalex.org/W2610189169","https://openalex.org/W2736601468","https://openalex.org/W2766074576","https://openalex.org/W2777467089","https://openalex.org/W2908457428","https://openalex.org/W3003820461","https://openalex.org/W3011898846","https://openalex.org/W3032398409","https://openalex.org/W3046330465","https://openalex.org/W3117407503","https://openalex.org/W3154705048","https://openalex.org/W3157610864","https://openalex.org/W3161084251","https://openalex.org/W3176563613","https://openalex.org/W3194871405","https://openalex.org/W4292969277","https://openalex.org/W6741002519"],"related_works":["https://openalex.org/W2357124094","https://openalex.org/W2387399993","https://openalex.org/W2389739210","https://openalex.org/W2348924972","https://openalex.org/W2365736347","https://openalex.org/W2047454415","https://openalex.org/W2070040999","https://openalex.org/W2387293848","https://openalex.org/W4387399830","https://openalex.org/W1994643058"],"abstract_inverted_index":{"Because":[0],"of":[1,22,55,69,74,102,110,138,158,175,183,247,265,281,291,296,324,334,378,384,397,412,420,452,458,471],"their":[2],"unique":[3],"adaptability,":[4],"flexibility,":[5],"and":[6,20,45,82,125,135,197,235,276,284,311,322,348,363,380,394,415,454,464],"robustness,":[7],"musculoskeletal":[8,104,142,177,249,269,346,399],"robotic":[9,25],"systems":[10],"are":[11,27,59,76,116,161,170,258,388],"regarded":[12],"potentially":[13],"as":[14,172,298,300],"next-generation":[15],"robots.":[16],"However,":[17,318],"motion":[18,100,133,173,285,350,392],"learning":[19,101,134,200,208,233,376,393],"generation":[21,136,395],"such":[23],"a":[24,33,231,345,352],"system":[26,178,326,347],"still":[28],"challenging.":[29],"This":[30],"paper":[31],"presents":[32],"neuromuscular":[34,355],"control":[35,152,283,336,356,386,413,446,466],"method,":[36,113],"namely,":[37,368],"TMS-PPO,":[38],"based":[39,64],"on":[40,65],"time-varying":[41,359],"muscle":[42,122,282,360],"synergy":[43,361],"(TMS)":[44,362],"proximal":[46],"policy":[47],"optimization":[48],"(PPO).":[49],"The":[50,72,88,144,207,375],"electromyogram":[51],"(EMG)":[52],"activation":[53,96,123],"signals":[54],"actual":[56],"human":[57,119,185,255,279,306,342],"motions":[58,457],"decomposed":[60],"to":[61,78,93,98,179,218,242,244,274,287,294,425,461],"obtain":[62],"TMSs":[63,92,115,169],"the":[66,70,80,86,91,103,108,111,114,132,151,155,159,167,176,181,184,194,199,205,245,248,268,278,289,292,316,325,332,335,341,381,385,391,398,418,421,426,441,445,450,456,459,468],"temporal":[67],"properties":[68],"TMS.":[71],"weights":[73],"networks":[75],"trained":[77],"generate":[79,94],"scale":[81],"phase":[83],"coefficients":[84,89],"through":[85],"PPO.":[87,206],"modulate":[90],"appropriate":[95],"patterns":[97],"optimize":[99],"system.":[105,143,400],"To":[106,431],"verify":[107],"effectiveness":[109],"proposed":[112,371],"extracted":[117],"from":[118,211],"upper":[120,140],"limb":[121,141],"signals,":[124],"we":[126,438],"compare":[127,288],"TMS-PPO":[128,148,192,229,369,422],"with":[129,204],"PPO":[130,379,427],"in":[131,302,315,372,410,435,467],"process":[137,182,387,470],"an":[139],"results":[145],"show":[146],"that":[147,191,228,253,295,357,405],"can":[149],"complete":[150],"tasks":[153],"because":[154],"average":[156],"errors":[157],"joints":[160],"less":[162],"than":[163],"0.05":[164],"rad.":[165],"In":[166],"meantime,":[168],"used":[171],"primitives":[174],"simulate":[180],"CNS":[186,343],"controlling":[187],"muscles.":[188],"It":[189],"shows":[190],"reduces":[193],"energy":[195],"consumption":[196],"improves":[198],"rate":[201],"significantly":[202],"compared":[203],"episodes":[209],"reduce":[210],"<inline-formula":[212,219],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[213,220,240],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">":[214,221],"<tex-math":[215,222],"notation=\"LaTeX\">$10^{4}$":[216],"</tex-math></inline-formula>":[217,224],"notation=\"LaTeX\">$10^{3}$":[223],",":[225],"which":[226,448],"indicates":[227],"has":[230],"stronger":[232],"ability":[234],"better":[236],"physiological":[237,382],"explanation.":[238],"<p":[239],"xmlns:xlink=\"http://www.w3.org/1999/xlink\">Note":[241],"Practitioners\u2014Due":[243],"superiorities":[246],"system,":[250],"humanoid":[251],"robots":[252,310,314],"imitate":[254],"driven":[256],"mechanisms":[257,280],"vigorously":[259],"carried":[260],"out":[261],"worldwide.":[262],"Taking":[263],"advantages":[264],"human-like":[266],"characteristics,":[267],"robot":[270,293],"provides":[271],"new":[272],"opportunities":[273],"understand":[275],"validate":[277],"learning,":[286],"performance":[290,419],"humans":[297],"well":[299],"work":[301],"real":[303],"world,":[304],"e.g.,":[305],"interactive":[307],"robots,":[308],"amusement":[309],"medical":[312],"training":[313],"future.":[317],"strong":[319],"redundancy,":[320],"coupling,":[321],"nonlinearity":[323],"also":[327],"raises":[328],"many":[329],"challenges":[330],"for":[331],"investigation":[333],"problem.":[337],"Inspired":[338],"by":[339],"how":[340],"controls":[344],"realize":[349],"generalization,":[351],"novel":[353],"muscle-synergies-based":[354],"combines":[358],"Proximal":[364],"Policy":[365],"Optimization":[366],"(PPO),":[367],"is":[370,408,423],"this":[373,406,433],"paper.":[374],"efficiency":[377],"interpretation":[383],"improved":[389],"during":[390],"processes":[396],"Preliminary":[401],"simulation":[402],"experiments":[403],"suggest":[404],"method":[407,447],"feasible":[409],"terms":[411],"accuracy":[414],"efficiency.":[416],"Moreover,":[417],"comparable":[424],"without":[428],"significant":[429],"improvement.":[430],"solve":[432],"problem,":[434],"future":[436],"work,":[437],"will":[439],"introduce":[440],"cerebellar":[442],"model":[443],"into":[444],"plays":[449],"role":[451],"adjusting":[453],"correcting":[455],"limbs":[460],"achieve":[462],"accurate":[463],"stable":[465],"actions":[469],"humans.":[472]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":9},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":3}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}