{"id":"https://openalex.org/W4389610330","doi":"https://doi.org/10.1109/tie.2023.3337547","title":"Hierarchical Reinforcement Learning-Based End-to-End Visual Servoing With Smooth Subgoals","display_name":"Hierarchical Reinforcement Learning-Based End-to-End Visual Servoing With Smooth Subgoals","publication_year":2023,"publication_date":"2023-12-12","ids":{"openalex":"https://openalex.org/W4389610330","doi":"https://doi.org/10.1109/tie.2023.3337547"},"language":"en","primary_location":{"id":"doi:10.1109/tie.2023.3337547","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tie.2023.3337547","pdf_url":null,"source":{"id":"https://openalex.org/S58031724","display_name":"IEEE Transactions on Industrial Electronics","issn_l":"0278-0046","issn":["0278-0046","1557-9948"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Industrial Electronics","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5014270991","display_name":"Yaozhen He","orcid":"https://orcid.org/0000-0002-4671-8371"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":true,"raw_author_name":"Yaozhen He","raw_affiliation_strings":["School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-4671-8371","affiliations":[{"raw_affiliation_string":"School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032200261","display_name":"Jian Gao","orcid":"https://orcid.org/0000-0002-1181-4531"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jian Gao","raw_affiliation_strings":["School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0002-1181-4531","affiliations":[{"raw_affiliation_string":"School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100605691","display_name":"Huiping Li","orcid":"https://orcid.org/0000-0003-4620-8993"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Huiping Li","raw_affiliation_strings":["School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0003-4620-8993","affiliations":[{"raw_affiliation_string":"School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100377728","display_name":"Yimin Chen","orcid":"https://orcid.org/0000-0003-1562-1443"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yimin Chen","raw_affiliation_strings":["School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0003-1562-1443","affiliations":[{"raw_affiliation_string":"School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070341037","display_name":"Yufeng Li","orcid":"https://orcid.org/0000-0003-0634-6734"},"institutions":[{"id":"https://openalex.org/I17145004","display_name":"Northwestern Polytechnical University","ror":"https://ror.org/01y0j0j86","country_code":"CN","type":"education","lineage":["https://openalex.org/I17145004"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yufeng Li","raw_affiliation_strings":["School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China"],"raw_orcid":"https://orcid.org/0000-0003-0634-6734","affiliations":[{"raw_affiliation_string":"School of Marine Science and Technology, Northwestern Polytechnical University, Xi'an, China","institution_ids":["https://openalex.org/I17145004"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":5,"corresponding_author_ids":["https://openalex.org/A5014270991"],"corresponding_institution_ids":["https://openalex.org/I17145004"],"apc_list":null,"apc_paid":null,"fwci":0.3532,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.61068623,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":96},"biblio":{"volume":"71","issue":"9","first_page":"11009","last_page":"11018"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10531","display_name":"Advanced Vision and Imaging","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10531","display_name":"Advanced Vision and Imaging","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10170","display_name":"Retinal Diseases and Treatments","score":0.9835000038146973,"subfield":{"id":"https://openalex.org/subfields/2731","display_name":"Ophthalmology"},"field":{"id":"https://openalex.org/fields/27","display_name":"Medicine"},"domain":{"id":"https://openalex.org/domains/4","display_name":"Health Sciences"}},{"id":"https://openalex.org/T10540","display_name":"Advanced Fluorescence Microscopy Techniques","score":0.9811999797821045,"subfield":{"id":"https://openalex.org/subfields/1304","display_name":"Biophysics"},"field":{"id":"https://openalex.org/fields/13","display_name":"Biochemistry, Genetics and Molecular Biology"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/smoothing","display_name":"Smoothing","score":0.7675598859786987},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7414548397064209},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.684810996055603},{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.6775615215301514},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6054926514625549},{"id":"https://openalex.org/keywords/visual-servoing","display_name":"Visual servoing","score":0.598037600517273},{"id":"https://openalex.org/keywords/end-to-end-principle","display_name":"End-to-end principle","score":0.5858230590820312},{"id":"https://openalex.org/keywords/process","display_name":"Process (computing)","score":0.525542676448822},{"id":"https://openalex.org/keywords/state","display_name":"State (computer science)","score":0.523724377155304},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.44858455657958984},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.35713452100753784},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.2758248448371887},{"id":"https://openalex.org/keywords/image","display_name":"Image (mathematics)","score":0.17814737558364868},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.13786527514457703}],"concepts":[{"id":"https://openalex.org/C3770464","wikidata":"https://www.wikidata.org/wiki/Q775963","display_name":"Smoothing","level":2,"score":0.7675598859786987},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7414548397064209},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.684810996055603},{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.6775615215301514},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6054926514625549},{"id":"https://openalex.org/C10912380","wikidata":"https://www.wikidata.org/wiki/Q527952","display_name":"Visual servoing","level":3,"score":0.598037600517273},{"id":"https://openalex.org/C74296488","wikidata":"https://www.wikidata.org/wiki/Q2527392","display_name":"End-to-end principle","level":2,"score":0.5858230590820312},{"id":"https://openalex.org/C98045186","wikidata":"https://www.wikidata.org/wiki/Q205663","display_name":"Process (computing)","level":2,"score":0.525542676448822},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.523724377155304},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.44858455657958984},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.35713452100753784},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.2758248448371887},{"id":"https://openalex.org/C115961682","wikidata":"https://www.wikidata.org/wiki/Q860623","display_name":"Image (mathematics)","level":2,"score":0.17814737558364868},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.13786527514457703},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C27206212","wikidata":"https://www.wikidata.org/wiki/Q34178","display_name":"Theology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tie.2023.3337547","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tie.2023.3337547","pdf_url":null,"source":{"id":"https://openalex.org/S58031724","display_name":"IEEE Transactions on Industrial Electronics","issn_l":"0278-0046","issn":["0278-0046","1557-9948"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Industrial Electronics","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G4363926683","display_name":null,"funder_award_id":"52102469","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6812378864","display_name":null,"funder_award_id":"51979228","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":47,"referenced_works":["https://openalex.org/W779494576","https://openalex.org/W2082991751","https://openalex.org/W2100249329","https://openalex.org/W2145339207","https://openalex.org/W2530520848","https://openalex.org/W2886946457","https://openalex.org/W2890755534","https://openalex.org/W2896248523","https://openalex.org/W2903253065","https://openalex.org/W2910687636","https://openalex.org/W2914261249","https://openalex.org/W2922466521","https://openalex.org/W2935417428","https://openalex.org/W2962894046","https://openalex.org/W2963523627","https://openalex.org/W2964067469","https://openalex.org/W3003329097","https://openalex.org/W3010515602","https://openalex.org/W3013461395","https://openalex.org/W3021493720","https://openalex.org/W3029795912","https://openalex.org/W3041800508","https://openalex.org/W3120718667","https://openalex.org/W3126840281","https://openalex.org/W3129297821","https://openalex.org/W3201577242","https://openalex.org/W3216509389","https://openalex.org/W4287867830","https://openalex.org/W4293497069","https://openalex.org/W4310851443","https://openalex.org/W4312944233","https://openalex.org/W4318606461","https://openalex.org/W4319996466","https://openalex.org/W4324125245","https://openalex.org/W4366371396","https://openalex.org/W4375928942","https://openalex.org/W6622487243","https://openalex.org/W6717230150","https://openalex.org/W6730641667","https://openalex.org/W6741002519","https://openalex.org/W6741471465","https://openalex.org/W6747473740","https://openalex.org/W6752089545","https://openalex.org/W6756303580","https://openalex.org/W6758978475","https://openalex.org/W6771807793","https://openalex.org/W6775522024"],"related_works":["https://openalex.org/W2126852585","https://openalex.org/W2367086100","https://openalex.org/W2381242807","https://openalex.org/W2352270872","https://openalex.org/W1520171478","https://openalex.org/W2386520554","https://openalex.org/W4317830657","https://openalex.org/W2127866683","https://openalex.org/W3126131230","https://openalex.org/W3024380338"],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1,45,115],"(RL)":[2],"offers":[3],"the":[4,30,46,58,84,106,114,129,132,148],"possibility":[5],"of":[6,10,60,131],"an":[7],"end-to-end":[8],"strategy":[9,48,70],"visual":[11,119],"servoing":[12],"(VS)":[13],"from":[14],"captured":[15],"images":[16],"or":[17],"features.":[18],"However,":[19],"there":[20],"will":[21],"be":[22],"unsmooth":[23],"actions":[24],"when":[25],"RL-agent":[26],"solely":[27],"depends":[28],"on":[29,50,57],"current":[31,85],"state.":[32],"In":[33],"this":[34],"article,":[35],"a":[36,68,73,99,125],"hierarchical":[37],"proximal":[38],"policy":[39,78,82],"optimization":[40],"method":[41],"is":[42,63,103],"proposed":[43,133],"for":[44,76,94,124],"VS":[47,139],"based":[49,56],"RL.":[51],"A":[52],"subgoal":[53,75,88],"generation":[54],"function":[55],"sequence":[59],"historical":[61,96],"data":[62],"designed":[64],"and":[65,87,138],"defined":[66],"as":[67,92],"high-level":[69],"to":[71,109,127],"provide":[72],"smooth":[74],"low-level":[77,81],"training.":[79],"The":[80,117,135],"takes":[83],"state":[86],"with":[89],"smoothing":[90],"attributes":[91],"inputs":[93],"considering":[95],"data.":[97],"Furthermore,":[98],"novel":[100],"measurement":[101],"approach":[102],"introduced":[104],"through":[105],"mean":[107],"cluster":[108],"encourage":[110],"agent":[111],"exploration":[112],"during":[113],"process.":[116],"autonomous":[118],"landing":[120],"experiments":[121],"are":[122,145],"conducted":[123],"quadrotor":[126],"validate":[128],"effectiveness":[130],"algorithm.":[134],"novelty":[136],"analysis":[137,141],"performance":[140],"in":[142,147],"different":[143],"scenarios":[144],"shown":[146],"comparative":[149],"experiments.":[150]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}