{"id":"https://openalex.org/W4312314355","doi":"https://doi.org/10.1109/tsmc.2022.3209070","title":"A Task-Agnostic Regularizer for Diverse Subpolicy Discovery in Hierarchical Reinforcement Learning","display_name":"A Task-Agnostic Regularizer for Diverse Subpolicy Discovery in Hierarchical Reinforcement Learning","publication_year":2022,"publication_date":"2022-09-30","ids":{"openalex":"https://openalex.org/W4312314355","doi":"https://doi.org/10.1109/tsmc.2022.3209070"},"language":"en","primary_location":{"id":"doi:10.1109/tsmc.2022.3209070","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmc.2022.3209070","pdf_url":null,"source":{"id":"https://openalex.org/S4210209078","display_name":"IEEE Transactions on Systems Man and Cybernetics Systems","issn_l":"2168-2216","issn":["2168-2216","2168-2232"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics: Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5045702955","display_name":"Liangyu Huo","orcid":"https://orcid.org/0000-0002-0703-3978"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liangyu Huo","raw_affiliation_strings":["School of Electronic and Information Engineering, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0703-3978","affiliations":[{"raw_affiliation_string":"School of Electronic and Information Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103046319","display_name":"Zulin Wang","orcid":"https://orcid.org/0000-0002-1328-7739"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zulin Wang","raw_affiliation_strings":["School of Electronic and Information Engineering, Beihang University, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electronic and Information Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079385534","display_name":"Mai Xu","orcid":"https://orcid.org/0000-0002-0277-3301"},"institutions":[{"id":"https://openalex.org/I82880672","display_name":"Beihang University","ror":"https://ror.org/00wk2mp56","country_code":"CN","type":"education","lineage":["https://openalex.org/I82880672"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Mai Xu","raw_affiliation_strings":["School of Electronic and Information Engineering, Beihang University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-0277-3301","affiliations":[{"raw_affiliation_string":"School of Electronic and Information Engineering, Beihang University, Beijing, China","institution_ids":["https://openalex.org/I82880672"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004775934","display_name":"Yuhang Song","orcid":"https://orcid.org/0000-0002-7999-0291"},"institutions":[{"id":"https://openalex.org/I40120149","display_name":"University of Oxford","ror":"https://ror.org/052gg0110","country_code":"GB","type":"education","lineage":["https://openalex.org/I40120149"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Yuhang Song","raw_affiliation_strings":["Department of Computer Science, University of Oxford, Oxford, U.K"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of Oxford, Oxford, U.K","institution_ids":["https://openalex.org/I40120149"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.3121,"has_fulltext":false,"cited_by_count":10,"citation_normalized_percentile":{"value":0.83672576,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":"53","issue":"3","first_page":"1932","last_page":"1944"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7814459800720215},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7594951391220093},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.662094235420227},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5858860015869141},{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.5264533162117004},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.4292324185371399},{"id":"https://openalex.org/keywords/hyperparameter","display_name":"Hyperparameter","score":0.417141854763031}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7814459800720215},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7594951391220093},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.662094235420227},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5858860015869141},{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.5264533162117004},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.4292324185371399},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.417141854763031},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tsmc.2022.3209070","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmc.2022.3209070","pdf_url":null,"source":{"id":"https://openalex.org/S4210209078","display_name":"IEEE Transactions on Systems Man and Cybernetics Systems","issn_l":"2168-2216","issn":["2168-2216","2168-2232"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics: Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G3168399153","display_name":null,"funder_award_id":"61876013","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G336532420","display_name":null,"funder_award_id":"61922009","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G388300612","display_name":null,"funder_award_id":"62250001","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G5585223276","display_name":null,"funder_award_id":"61925105","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G6127711580","display_name":null,"funder_award_id":"62231002","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":43,"referenced_works":["https://openalex.org/W1821462560","https://openalex.org/W1987725948","https://openalex.org/W2099471712","https://openalex.org/W2109910161","https://openalex.org/W2145339207","https://openalex.org/W2158782408","https://openalex.org/W2740985535","https://openalex.org/W2964118262","https://openalex.org/W2964227312","https://openalex.org/W2967043083","https://openalex.org/W2970999977","https://openalex.org/W3016244212","https://openalex.org/W3040408533","https://openalex.org/W3115597439","https://openalex.org/W3168892396","https://openalex.org/W4214717370","https://openalex.org/W6629461247","https://openalex.org/W6634423325","https://openalex.org/W6638803421","https://openalex.org/W6674884181","https://openalex.org/W6683821272","https://openalex.org/W6684921986","https://openalex.org/W6692846177","https://openalex.org/W6729130418","https://openalex.org/W6734215269","https://openalex.org/W6736057607","https://openalex.org/W6736368053","https://openalex.org/W6741002519","https://openalex.org/W6744935223","https://openalex.org/W6745347688","https://openalex.org/W6747473740","https://openalex.org/W6748566876","https://openalex.org/W6748600884","https://openalex.org/W6748603076","https://openalex.org/W6749821205","https://openalex.org/W6755459604","https://openalex.org/W6760698134","https://openalex.org/W6764724164","https://openalex.org/W6766234440","https://openalex.org/W6790829992","https://openalex.org/W6796667233","https://openalex.org/W6797416203","https://openalex.org/W6810265657"],"related_works":["https://openalex.org/W4281847915","https://openalex.org/W3000635674","https://openalex.org/W2602382373","https://openalex.org/W3003615511","https://openalex.org/W4285827128","https://openalex.org/W3198113463","https://openalex.org/W2787698406","https://openalex.org/W2963844355","https://openalex.org/W4361251046","https://openalex.org/W98577079"],"abstract_inverted_index":{"The":[0,146],"automatic":[1],"subpolicy":[2,48,144],"discovery":[3,82],"approach":[4,154],"in":[5,75,137],"hierarchical":[6,134],"reinforcement":[7,131],"learning":[8,22,72,136],"(HRL)":[9],"has":[10],"recently":[11],"achieved":[12],"promising":[13],"performance":[14,160],"on":[15,124,161],"sparse":[16],"reward":[17],"tasks.":[18],"This":[19],"accelerates":[20],"transfer":[21],"and":[23,91,113,141,175],"unsupervised":[24,143],"intelligent":[25],"creatures":[26],"while":[27],"eliminating":[28],"the":[29,44,50,57,81,106,120,138,158,172],"domain-specific":[30],"knowledge":[31],"constraint.":[32],"Most":[33],"previously":[34],"developed":[35],"approaches":[36],"are":[37],"demonstrated":[38],"to":[39,98],"suffer":[40],"from":[41],"collapsing":[42],"into":[43],"situation":[45],"where":[46],"one":[47],"dominates":[49],"whole":[51],"task,":[52],"since":[53],"they":[54],"cannot":[55],"ensure":[56],"diversity":[58],"of":[59,83,177],"different":[60],"subpolicies.":[61],"In":[62],"contrast,":[63],"this":[64],"article":[65],"proposes":[66],"a":[67,87,94],"task-agnostic":[68],"regularizer":[69,123],"(TAR)":[70],"for":[71,111],"diverse":[73,84],"subpolicies":[74,85],"HRL.":[76],"Specifically,":[77],"we":[78,104],"first":[79],"formulate":[80],"as":[86,108],"trajectory":[88],"inference":[89],"problem":[90],"then":[92],"propose":[93],"corresponding":[95],"information-theoretic":[96],"objective":[97,107],"encourage":[99],"diversity.":[100],"Subsequently,":[101],"considering":[102],"computability,":[103],"instantiate":[105],"two":[109],"simplifications":[110],"discrete":[112],"continuous":[114],"action":[115],"spaces.":[116],"We":[117],"extensively":[118],"evaluate":[119],"proposed":[121],"diversity-driven":[122],"three":[125,163],"HRL":[126,164],"task":[127],"domains:":[128],"1)":[129],"meta":[130],"learning;":[132],"2)":[133],"policy":[135],"option":[139],"framework;":[140],"3)":[142],"discovery.":[145],"extensive":[147],"results":[148],"obtained":[149],"show":[150],"that":[151],"our":[152,178],"TAR":[153],"can":[155],"improve":[156],"upon":[157],"state-of-the-art":[159],"all":[162],"domains":[165],"without":[166],"modifying":[167],"any":[168],"existing":[169],"hyperparameters,":[170],"indicating":[171],"wide":[173],"applicability":[174],"robustness":[176],"approach.":[179]},"counts_by_year":[{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}