{"id":"https://openalex.org/W4411232152","doi":"https://doi.org/10.1109/tsmc.2025.3574024","title":"Exploration via Embracing Diversity in Reinforcement Learning for Sparse-Reward Procedurally-Generated Tasks","display_name":"Exploration via Embracing Diversity in Reinforcement Learning for Sparse-Reward Procedurally-Generated Tasks","publication_year":2025,"publication_date":"2025-06-12","ids":{"openalex":"https://openalex.org/W4411232152","doi":"https://doi.org/10.1109/tsmc.2025.3574024"},"language":"en","primary_location":{"id":"doi:10.1109/tsmc.2025.3574024","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmc.2025.3574024","pdf_url":null,"source":{"id":"https://openalex.org/S4210209078","display_name":"IEEE Transactions on Systems Man and Cybernetics Systems","issn_l":"2168-2216","issn":["2168-2216","2168-2232"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics: Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5101814011","display_name":"Pei Xu","orcid":"https://orcid.org/0000-0003-1766-5634"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Pei Xu","raw_affiliation_strings":["Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","Institute of Automation, Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0003-1766-5634","affiliations":[{"raw_affiliation_string":"Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210094879","https://openalex.org/I19820366"]},{"raw_affiliation_string":"Institute of Automation, Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210094879","https://openalex.org/I19820366"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103169234","display_name":"Hao Chen","orcid":"https://orcid.org/0000-0002-5982-0615"},"institutions":[{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Chen","raw_affiliation_strings":["School of Emergency Management Science and Engineering, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-5982-0615","affiliations":[{"raw_affiliation_string":"School of Emergency Management Science and Engineering, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210165038"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101736643","display_name":"Wenjie Yang","orcid":"https://orcid.org/0000-0001-7483-1817"},"institutions":[{"id":"https://openalex.org/I80947539","display_name":"Fuzhou University","ror":"https://ror.org/011xvna82","country_code":"CN","type":"education","lineage":["https://openalex.org/I80947539"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Wenjie Yang","raw_affiliation_strings":["College of Computer and Data Science, Fuzhou University, Fuzhou, China"],"raw_orcid":"https://orcid.org/0000-0001-7483-1817","affiliations":[{"raw_affiliation_string":"College of Computer and Data Science, Fuzhou University, Fuzhou, China","institution_ids":["https://openalex.org/I80947539"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028693655","display_name":"Kaiqi Huang","orcid":"https://orcid.org/0000-0002-2677-9273"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210094879","display_name":"Shandong Institute of Automation","ror":"https://ror.org/00qdtba35","country_code":"CN","type":"facility","lineage":["https://openalex.org/I4210094879","https://openalex.org/I4210142748"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kaiqi Huang","raw_affiliation_strings":["Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","Institute of Automation, Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-2677-9273","affiliations":[{"raw_affiliation_string":"Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210094879","https://openalex.org/I19820366"]},{"raw_affiliation_string":"Institute of Automation, Key Laboratory of Cognition and Decision Intelligence for Complex Systems, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210094879","https://openalex.org/I19820366"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":1.067,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.7665095,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":98},"biblio":{"volume":"55","issue":"9","first_page":"5776","last_page":"5789"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9358999729156494,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10429","display_name":"EEG and Brain-Computer Interfaces","score":0.9358999729156494,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.699928879737854},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6316056251525879},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.5370304584503174},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.45400097966194153},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.42115652561187744},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.40886902809143066},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.24455907940864563},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.16387856006622314},{"id":"https://openalex.org/keywords/sociology","display_name":"Sociology","score":0.08334800601005554}],"concepts":[{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.699928879737854},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6316056251525879},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.5370304584503174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.45400097966194153},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.42115652561187744},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.40886902809143066},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.24455907940864563},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.16387856006622314},{"id":"https://openalex.org/C144024400","wikidata":"https://www.wikidata.org/wiki/Q21201","display_name":"Sociology","level":0,"score":0.08334800601005554},{"id":"https://openalex.org/C19165224","wikidata":"https://www.wikidata.org/wiki/Q23404","display_name":"Anthropology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tsmc.2025.3574024","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tsmc.2025.3574024","pdf_url":null,"source":{"id":"https://openalex.org/S4210209078","display_name":"IEEE Transactions on Systems Man and Cybernetics Systems","issn_l":"2168-2216","issn":["2168-2216","2168-2232"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Systems, Man, and Cybernetics: Systems","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.49000000953674316,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G6464566889","display_name":null,"funder_award_id":"GZC20232995","funder_id":"https://openalex.org/F4320321543","funder_display_name":"China Postdoctoral Science Foundation"},{"id":"https://openalex.org/G7221713726","display_name":null,"funder_award_id":"2022ZD0116403","funder_id":"https://openalex.org/F4320329860","funder_display_name":"National Science and Technology Major Project"}],"funders":[{"id":"https://openalex.org/F4320321543","display_name":"China Postdoctoral Science Foundation","ror":"https://ror.org/0426zh255"},{"id":"https://openalex.org/F4320329860","display_name":"National Science and Technology Major Project","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":50,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W952777547","https://openalex.org/W1988526405","https://openalex.org/W2119717200","https://openalex.org/W2145339207","https://openalex.org/W2150468603","https://openalex.org/W2164943005","https://openalex.org/W2746553466","https://openalex.org/W2761873684","https://openalex.org/W2810602713","https://openalex.org/W2963523627","https://openalex.org/W2966556569","https://openalex.org/W2982316857","https://openalex.org/W3045372678","https://openalex.org/W3165885159","https://openalex.org/W4205779441","https://openalex.org/W4312314355","https://openalex.org/W4382202977","https://openalex.org/W4385767435","https://openalex.org/W4401024104","https://openalex.org/W6608012839","https://openalex.org/W6638018090","https://openalex.org/W6683195989","https://openalex.org/W6683204974","https://openalex.org/W6685444567","https://openalex.org/W6687681856","https://openalex.org/W6692846177","https://openalex.org/W6717230150","https://openalex.org/W6734129231","https://openalex.org/W6735033012","https://openalex.org/W6740092555","https://openalex.org/W6741002519","https://openalex.org/W6748523217","https://openalex.org/W6751540476","https://openalex.org/W6752431149","https://openalex.org/W6752725515","https://openalex.org/W6753925943","https://openalex.org/W6754957883","https://openalex.org/W6756303580","https://openalex.org/W6757058172","https://openalex.org/W6771750343","https://openalex.org/W6771807793","https://openalex.org/W6771968188","https://openalex.org/W6779637449","https://openalex.org/W6788860121","https://openalex.org/W6789598788","https://openalex.org/W6797415283","https://openalex.org/W6803660280","https://openalex.org/W6846255677","https://openalex.org/W6922480057"],"related_works":["https://openalex.org/W4391375266","https://openalex.org/W2748952813","https://openalex.org/W4310083477","https://openalex.org/W2328553770","https://openalex.org/W2920061524","https://openalex.org/W1977959518","https://openalex.org/W2038908348","https://openalex.org/W2107890255","https://openalex.org/W2106552856","https://openalex.org/W2145821588"],"abstract_inverted_index":{"A":[0],"key":[1],"challenge":[2],"in":[3,45,54,119,183],"reinforcement":[4],"learning":[5],"is":[6,52,61],"how":[7],"to":[8,11,19,64,93,115,135,154,167],"guide":[9],"agents":[10],"efficiently":[12],"explore":[13],"sparse":[14],"reward":[15],"environments.":[16],"In":[17,97,122,158],"order":[18],"overcome":[20],"this":[21,98,156],"challenge,":[22],"the":[23,36,59,66,137,140,145,152,165,169,206],"state-of-the-art":[24,196],"methods":[25,42,75,197],"introduce":[26],"additional":[27],"intrinsic":[28],"rewards":[29],"based":[30],"on":[31,198],"state-related":[32,89,113,161],"information,":[33,90,126,162],"such":[34],"as":[35,177,181],"novelty":[37],"of":[38,124,160,209],"states.":[39],"Unfortunately,":[40],"these":[41],"frequently":[43],"fail":[44],"procedurally-generated":[46,79,120],"tasks,":[47],"where":[48],"a":[49,102],"different":[50,179],"environment":[51],"generated":[53],"each":[55],"episode":[56],"so":[57],"that":[58,188],"agent":[60,153,166],"not":[62],"likely":[63],"visit":[65],"same":[67],"state":[68,170],"more":[69],"than":[70],"once.":[71],"Recently,":[72],"some":[73],"exploration":[74,104,118],"designed":[76],"specifically":[77],"for":[78],"tasks":[80],"have":[81],"been":[82],"proposed.":[83],"However,":[84],"they":[85],"still":[86],"only":[87],"consider":[88],"which":[91,106],"leads":[92],"relatively":[94],"inefficient":[95],"exploration.":[96],"work,":[99],"we":[100,127,163],"propose":[101],"novel":[103],"method,":[105],"utilizes":[107],"cross-episode":[108],"policy-related":[109,125],"information":[110,114],"and":[111,144,149,205],"intraepisode":[112],"jointly":[116],"encourage":[117,151,164],"tasks.":[121],"term":[123,159],"first":[128],"use":[129],"an":[130,173,184],"imitator-based":[131],"unbalanced":[132],"policy":[133,143],"diversity":[134,171],"measure":[136],"difference":[138],"between":[139],"agent\u2019s":[141,146],"current":[142],"previous":[147],"policies,":[148],"then":[150],"maximize":[155,168],"difference.":[157],"within":[172],"episode,":[174],"thereby":[175],"visiting":[176],"many":[178],"states":[180],"possible":[182],"episode.":[185],"We":[186],"show":[187],"our":[189],"method":[190],"significantly":[191],"improves":[192],"sample":[193],"efficiency":[194],"over":[195],"three":[199],"challenging":[200],"benchmarks,":[201],"including":[202],"MiniGrid,":[203],"MiniWorld,":[204],"sparse-reward":[207],"version":[208],"Procgen.":[210]},"counts_by_year":[{"year":2026,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}