{"id":"https://openalex.org/W4415883067","doi":"https://doi.org/10.1109/lcsys.2025.3629008","title":"Feedback for Improved Hierarchical Reinforcement Learning With Timed Subgoals","display_name":"Feedback for Improved Hierarchical Reinforcement Learning With Timed Subgoals","publication_year":2025,"publication_date":"2025-01-01","ids":{"openalex":"https://openalex.org/W4415883067","doi":"https://doi.org/10.1109/lcsys.2025.3629008"},"language":null,"primary_location":{"id":"doi:10.1109/lcsys.2025.3629008","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lcsys.2025.3629008","pdf_url":null,"source":{"id":"https://openalex.org/S4306422535","display_name":"IEEE Control Systems Letters","issn_l":"2475-1456","issn":["2475-1456"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Control Systems Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5058947568","display_name":"Yajie Bao","orcid":"https://orcid.org/0000-0001-8773-926X"},"institutions":[{"id":"https://openalex.org/I4210131102","display_name":"Intelligent Fusion Technology (United States)","ror":"https://ror.org/02hjcc687","country_code":"US","type":"company","lineage":["https://openalex.org/I4210131102"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yajie Bao","raw_affiliation_strings":["Department of Research and Development, Intelligent Fusion Technology, Inc., Germantown, MD, USA","Intelligent Fusion Technology, Inc, Germantown, MD, USA"],"raw_orcid":"https://orcid.org/0000-0001-8773-926X","affiliations":[{"raw_affiliation_string":"Department of Research and Development, Intelligent Fusion Technology, Inc., Germantown, MD, USA","institution_ids":["https://openalex.org/I4210131102"]},{"raw_affiliation_string":"Intelligent Fusion Technology, Inc, Germantown, MD, USA","institution_ids":["https://openalex.org/I4210131102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004808253","display_name":"Dan Shen","orcid":"https://orcid.org/0000-0003-1834-5456"},"institutions":[{"id":"https://openalex.org/I4210131102","display_name":"Intelligent Fusion Technology (United States)","ror":"https://ror.org/02hjcc687","country_code":"US","type":"company","lineage":["https://openalex.org/I4210131102"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dan Shen","raw_affiliation_strings":["Department of Research and Development, Intelligent Fusion Technology, Inc., Germantown, MD, USA","Intelligent Fusion Technology, Inc, Germantown, MD, USA"],"raw_orcid":"https://orcid.org/0000-0003-1834-5456","affiliations":[{"raw_affiliation_string":"Department of Research and Development, Intelligent Fusion Technology, Inc., Germantown, MD, USA","institution_ids":["https://openalex.org/I4210131102"]},{"raw_affiliation_string":"Intelligent Fusion Technology, Inc, Germantown, MD, USA","institution_ids":["https://openalex.org/I4210131102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5038601007","display_name":"Genshe Chen","orcid":"https://orcid.org/0000-0002-2656-9380"},"institutions":[{"id":"https://openalex.org/I4210131102","display_name":"Intelligent Fusion Technology (United States)","ror":"https://ror.org/02hjcc687","country_code":"US","type":"company","lineage":["https://openalex.org/I4210131102"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Genshe Chen","raw_affiliation_strings":["Department of Research and Development, Intelligent Fusion Technology, Inc., Germantown, MD, USA","Intelligent Fusion Technology, Inc, Germantown, MD, USA"],"raw_orcid":"https://orcid.org/0000-0002-2656-9380","affiliations":[{"raw_affiliation_string":"Department of Research and Development, Intelligent Fusion Technology, Inc., Germantown, MD, USA","institution_ids":["https://openalex.org/I4210131102"]},{"raw_affiliation_string":"Intelligent Fusion Technology, Inc, Germantown, MD, USA","institution_ids":["https://openalex.org/I4210131102"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100717012","display_name":"Hao Xu","orcid":"https://orcid.org/0000-0003-4130-7925"},"institutions":[{"id":"https://openalex.org/I134113660","display_name":"University of Nevada, Reno","ror":"https://ror.org/01keh0577","country_code":"US","type":"education","lineage":["https://openalex.org/I134113660"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hao Xu","raw_affiliation_strings":["Department of Electrical and Biomedical Engineering, University of Nevada, Reno, NV, USA"],"raw_orcid":"https://orcid.org/0000-0003-4130-7925","affiliations":[{"raw_affiliation_string":"Department of Electrical and Biomedical Engineering, University of Nevada, Reno, NV, USA","institution_ids":["https://openalex.org/I134113660"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120251909","display_name":"Samson Badlia","orcid":null},"institutions":[{"id":"https://openalex.org/I1280414376","display_name":"United States Air Force Research Laboratory","ror":"https://ror.org/02e2egq70","country_code":"US","type":"facility","lineage":["https://openalex.org/I1280414376","https://openalex.org/I1330347796","https://openalex.org/I4210102105","https://openalex.org/I4389425425"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Samson Badlia","raw_affiliation_strings":["Advanced Planning and Autonomous C2 System Branch, Air Force Research Laboratory, Rome, NY, USA","Air Force Research Laboratory, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Advanced Planning and Autonomous C2 System Branch, Air Force Research Laboratory, Rome, NY, USA","institution_ids":["https://openalex.org/I1280414376"]},{"raw_affiliation_string":"Air Force Research Laboratory, USA","institution_ids":["https://openalex.org/I1280414376"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5065073687","display_name":"Simon Khan","orcid":"https://orcid.org/0000-0002-0096-8682"},"institutions":[{"id":"https://openalex.org/I1280414376","display_name":"United States Air Force Research Laboratory","ror":"https://ror.org/02e2egq70","country_code":"US","type":"facility","lineage":["https://openalex.org/I1280414376","https://openalex.org/I1330347796","https://openalex.org/I4210102105","https://openalex.org/I4389425425"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Simon Khan","raw_affiliation_strings":["Advanced Planning and Autonomous C2 System Branch, Air Force Research Laboratory, Rome, NY, USA","Air Force Research Laboratory, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Advanced Planning and Autonomous C2 System Branch, Air Force Research Laboratory, Rome, NY, USA","institution_ids":["https://openalex.org/I1280414376"]},{"raw_affiliation_string":"Air Force Research Laboratory, USA","institution_ids":["https://openalex.org/I1280414376"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5023894377","display_name":"Erik Blasch","orcid":"https://orcid.org/0000-0001-6894-6108"},"institutions":[{"id":"https://openalex.org/I1280414376","display_name":"United States Air Force Research Laboratory","ror":"https://ror.org/02e2egq70","country_code":"US","type":"facility","lineage":["https://openalex.org/I1280414376","https://openalex.org/I1330347796","https://openalex.org/I4210102105","https://openalex.org/I4389425425"]},{"id":"https://openalex.org/I4210099573","display_name":"United States Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93","country_code":"US","type":"government","lineage":["https://openalex.org/I1280414376","https://openalex.org/I1330347796","https://openalex.org/I4210099573","https://openalex.org/I4210102105","https://openalex.org/I4389425425"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Erik Blasch","raw_affiliation_strings":["Air Force Office of Scientific Research, Air Force Research Laboratory, Arlington, VA, USA","Air Force Research Laboratory, USA"],"raw_orcid":"https://orcid.org/0000-0001-6894-6108","affiliations":[{"raw_affiliation_string":"Air Force Office of Scientific Research, Air Force Research Laboratory, Arlington, VA, USA","institution_ids":["https://openalex.org/I4210099573"]},{"raw_affiliation_string":"Air Force Research Laboratory, USA","institution_ids":["https://openalex.org/I1280414376"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017878875","display_name":"Khanh Pham","orcid":"https://orcid.org/0000-0002-4768-9365"},"institutions":[{"id":"https://openalex.org/I1280414376","display_name":"United States Air Force Research Laboratory","ror":"https://ror.org/02e2egq70","country_code":"US","type":"facility","lineage":["https://openalex.org/I1280414376","https://openalex.org/I1330347796","https://openalex.org/I4210102105","https://openalex.org/I4389425425"]},{"id":"https://openalex.org/I2800117799","display_name":"Kirtland Air Force Base","ror":"https://ror.org/04erk7g84","country_code":"US","type":"other","lineage":["https://openalex.org/I1330347796","https://openalex.org/I2800117799","https://openalex.org/I4210089612","https://openalex.org/I4210102105"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Khanh Pham","raw_affiliation_strings":["Space Vehicle Directorate, Air Force Research Laboratory, Kirtland Air Force Base, New Mexico, USA","Air Force Research Laboratory, USA"],"raw_orcid":"https://orcid.org/0000-0002-4768-9365","affiliations":[{"raw_affiliation_string":"Space Vehicle Directorate, Air Force Research Laboratory, Kirtland Air Force Base, New Mexico, USA","institution_ids":["https://openalex.org/I2800117799"]},{"raw_affiliation_string":"Air Force Research Laboratory, USA","institution_ids":["https://openalex.org/I1280414376"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5058947568"],"corresponding_institution_ids":["https://openalex.org/I4210131102"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.16237241,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"9","issue":null,"first_page":"2501","last_page":"2506"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8072999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8072999715805054,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.02239999920129776,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12611","display_name":"Neural Networks and Reservoir Computing","score":0.016599999740719795,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hindsight-bias","display_name":"Hindsight bias","score":0.7081999778747559},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6444000005722046},{"id":"https://openalex.org/keywords/set","display_name":"Set (abstract data type)","score":0.6399000287055969},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5418999791145325},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.5374000072479248},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.49709999561309814},{"id":"https://openalex.org/keywords/sample-complexity","display_name":"Sample complexity","score":0.45730000734329224},{"id":"https://openalex.org/keywords/feedback-control","display_name":"Feedback control","score":0.4250999987125397}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7524999976158142},{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.7081999778747559},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6444000005722046},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.6399000287055969},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5418999791145325},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.5374000072479248},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.49709999561309814},{"id":"https://openalex.org/C2778445095","wikidata":"https://www.wikidata.org/wiki/Q18354077","display_name":"Sample complexity","level":2,"score":0.45730000734329224},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4372999966144562},{"id":"https://openalex.org/C3018651601","wikidata":"https://www.wikidata.org/wiki/Q183635","display_name":"Feedback control","level":2,"score":0.4250999987125397},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.3686000108718872},{"id":"https://openalex.org/C77618280","wikidata":"https://www.wikidata.org/wiki/Q1155772","display_name":"Scheme (mathematics)","level":2,"score":0.364300012588501},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.32679998874664307},{"id":"https://openalex.org/C3017962763","wikidata":"https://www.wikidata.org/wiki/Q427492","display_name":"Feedback regulation","level":2,"score":0.29269999265670776},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2773999869823456},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C3018623182","wikidata":"https://www.wikidata.org/wiki/Q154021","display_name":"Output feedback","level":3,"score":0.2612999975681305},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.2597000002861023},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.2565999925136566},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.2538999915122986},{"id":"https://openalex.org/C112758219","wikidata":"https://www.wikidata.org/wiki/Q16038819","display_name":"Duration (music)","level":2,"score":0.2531999945640564}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lcsys.2025.3629008","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lcsys.2025.3629008","pdf_url":null,"source":{"id":"https://openalex.org/S4306422535","display_name":"IEEE Control Systems Letters","issn_l":"2475-1456","issn":["2475-1456"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Control Systems Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":8,"referenced_works":["https://openalex.org/W3192660112","https://openalex.org/W4205689302","https://openalex.org/W4382318696","https://openalex.org/W4401507809","https://openalex.org/W4406890416","https://openalex.org/W4407168487","https://openalex.org/W4410771875","https://openalex.org/W4413945492"],"related_works":[],"abstract_inverted_index":{"Hierarchical":[0],"reinforcement":[1],"learning":[2],"(RL)":[3],"aims":[4],"to":[5,70,110,117],"improve":[6],"sample":[7,143],"efficiency":[8,144],"by":[9,68,74],"decomposing":[10],"complex":[11],"long-horizon":[12],"tasks":[13],"into":[14],"fast":[15],"low-level":[16,71],"myopic":[17],"and":[18,58,113],"slower":[19],"high-level":[20,55],"non-myopic":[21],"subtasks.":[22],"However,":[23],"the":[24,39,46,52,107,114],"unilateral":[25],"nested":[26],"policy":[27],"structure":[28],"in":[29,126],"current":[30,102],"goal-conditioned":[31],"hierarchical":[32],"RL":[33,130],"(HRL)":[34],"methods":[35],"sets":[36],"subgoals":[37,112],"at":[38],"high":[40,108],"level":[41,109,116],"without":[42],"considering":[43],"feedback":[44,75,93],"from":[45],"low":[47,115],"level,":[48],"which":[49],"significantly":[50],"degrades":[51],"performance":[53],"of":[54,77,95],"subgoal":[56,66,96,122],"generation":[57,97],"sampling":[59],"efficiency.":[60],"Hindsight":[61],"action":[62],"relabeling":[63],"further":[64],"weakens":[65],"settings":[67],"submitting":[69],"reachability.":[72],"Inspired":[73],"control":[76,94],"dynamic":[78,129],"systems,":[79],"we":[80],"present":[81],"Feedback":[82],"for":[83,98],"Improved":[84],"HRL":[85,148],"with":[86,141],"Timed":[87],"Subgoals":[88],"(FIHTS),":[89],"a":[90],"mechanism":[91],"allowing":[92],"improved":[99],"HRL.":[100],"Unlike":[101],"HRL,":[103],"FIHTS":[104,135],"enables":[105],"both":[106],"set":[111],"receive":[118],"rewards":[119],"based":[120],"on":[121],"achievement.":[123],"Our":[124],"experiments":[125],"various":[127],"challenging":[128],"environments":[131],"show":[132],"that":[133],"our":[134],"method":[136],"achieves":[137],"higher":[138,142],"success":[139],"rates":[140],"than":[145],"existing":[146],"subgoal-based":[147],"methods.":[149]},"counts_by_year":[],"updated_date":"2026-05-04T08:30:34.212998","created_date":"2025-11-04T00:00:00"}