{"id":"https://openalex.org/W4308783088","doi":"https://doi.org/10.1109/lra.2022.3214057","title":"APD: Learning Diverse Behaviors for Reinforcement Learning Through Unsupervised Active Pre-Training","display_name":"APD: Learning Diverse Behaviors for Reinforcement Learning Through Unsupervised Active Pre-Training","publication_year":2022,"publication_date":"2022-10-01","ids":{"openalex":"https://openalex.org/W4308783088","doi":"https://doi.org/10.1109/lra.2022.3214057"},"language":"en","primary_location":{"id":"doi:10.1109/lra.2022.3214057","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2022.3214057","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5029359556","display_name":"Kailin Zeng","orcid":"https://orcid.org/0000-0001-5868-8758"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Kailin Zeng","raw_affiliation_strings":["School of Artificial Intelligence, Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5868-8758","affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100739520","display_name":"Qiyuan Zhang","orcid":"https://orcid.org/0000-0002-8519-4259"},"institutions":[{"id":"https://openalex.org/I204983213","display_name":"Harbin Institute of Technology","ror":"https://ror.org/01yqg2h08","country_code":"CN","type":"education","lineage":["https://openalex.org/I204983213"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"QiYuan Zhang","raw_affiliation_strings":["School of Mechatronics Engineering, Harbin Institute of Technology, Harbin, China"],"raw_orcid":"https://orcid.org/0000-0002-8519-4259","affiliations":[{"raw_affiliation_string":"School of Mechatronics Engineering, Harbin Institute of Technology, Harbin, China","institution_ids":["https://openalex.org/I204983213"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100427396","display_name":"Bin Chen","orcid":"https://orcid.org/0000-0003-0069-6565"},"institutions":[{"id":"https://openalex.org/I139759216","display_name":"Beijing University of Posts and Telecommunications","ror":"https://ror.org/04w9fbh59","country_code":"CN","type":"education","lineage":["https://openalex.org/I139759216"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Chen","raw_affiliation_strings":["School of Artificial Intelligence, Beijing University of Posts and Telecommunications, Beijing, China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Artificial Intelligence, Beijing University of Posts and Telecommunications, Beijing, China","institution_ids":["https://openalex.org/I139759216"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100644630","display_name":"Bin Liang","orcid":"https://orcid.org/0000-0002-8591-8843"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Bin Liang","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-8591-8843","affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101912623","display_name":"Jun Yang","orcid":"https://orcid.org/0000-0002-9386-5825"},"institutions":[{"id":"https://openalex.org/I99065089","display_name":"Tsinghua University","ror":"https://ror.org/03cve4549","country_code":"CN","type":"education","lineage":["https://openalex.org/I99065089"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Jun Yang","raw_affiliation_strings":["Department of Automation, Tsinghua University, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0002-9386-5825","affiliations":[{"raw_affiliation_string":"Department of Automation, Tsinghua University, Beijing, China","institution_ids":["https://openalex.org/I99065089"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.348,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.65516851,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":96,"max":97},"biblio":{"volume":"7","issue":"4","first_page":"12251","last_page":"12258"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9965999722480774,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9596999883651733,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9595000147819519,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7919463515281677},{"id":"https://openalex.org/keywords/discriminator","display_name":"Discriminator","score":0.709075391292572},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7052256464958191},{"id":"https://openalex.org/keywords/unsupervised-learning","display_name":"Unsupervised learning","score":0.694006621837616},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6648831367492676},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.6417875289916992},{"id":"https://openalex.org/keywords/entropy","display_name":"Entropy (arrow of time)","score":0.4632934331893921},{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.4428136944770813},{"id":"https://openalex.org/keywords/state-space","display_name":"State space","score":0.44142892956733704},{"id":"https://openalex.org/keywords/active-learning","display_name":"Active learning (machine learning)","score":0.434203565120697},{"id":"https://openalex.org/keywords/learning-classifier-system","display_name":"Learning classifier system","score":0.4115932881832123},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.09543737769126892}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7919463515281677},{"id":"https://openalex.org/C2779803651","wikidata":"https://www.wikidata.org/wiki/Q5282088","display_name":"Discriminator","level":3,"score":0.709075391292572},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7052256464958191},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.694006621837616},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6648831367492676},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.6417875289916992},{"id":"https://openalex.org/C106301342","wikidata":"https://www.wikidata.org/wiki/Q4117933","display_name":"Entropy (arrow of time)","level":2,"score":0.4632934331893921},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.4428136944770813},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.44142892956733704},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.434203565120697},{"id":"https://openalex.org/C199190896","wikidata":"https://www.wikidata.org/wiki/Q3509276","display_name":"Learning classifier system","level":3,"score":0.4115932881832123},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09543737769126892},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C76155785","wikidata":"https://www.wikidata.org/wiki/Q418","display_name":"Telecommunications","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C94915269","wikidata":"https://www.wikidata.org/wiki/Q1834857","display_name":"Detector","level":2,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/lra.2022.3214057","is_oa":false,"landing_page_url":"https://doi.org/10.1109/lra.2022.3214057","pdf_url":null,"source":{"id":"https://openalex.org/S4210169774","display_name":"IEEE Robotics and Automation Letters","issn_l":"2377-3766","issn":["2377-3766","2377-3774"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Robotics and Automation Letters","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/10","score":0.699999988079071,"display_name":"Reduced inequalities"}],"awards":[{"id":"https://openalex.org/G3309936734","display_name":null,"funder_award_id":"62073033","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":33,"referenced_works":["https://openalex.org/W41554520","https://openalex.org/W2139137304","https://openalex.org/W2145339207","https://openalex.org/W2953326529","https://openalex.org/W2963007936","https://openalex.org/W2963523627","https://openalex.org/W3015437096","https://openalex.org/W3036619998","https://openalex.org/W3206005330","https://openalex.org/W4206033892","https://openalex.org/W4224130159","https://openalex.org/W6676077707","https://openalex.org/W6684921986","https://openalex.org/W6685444567","https://openalex.org/W6716474083","https://openalex.org/W6717230150","https://openalex.org/W6730641667","https://openalex.org/W6735033012","https://openalex.org/W6740801417","https://openalex.org/W6745983339","https://openalex.org/W6748603076","https://openalex.org/W6764281707","https://openalex.org/W6765240361","https://openalex.org/W6773447848","https://openalex.org/W6780559895","https://openalex.org/W6790486821","https://openalex.org/W6791000347","https://openalex.org/W6791194670","https://openalex.org/W6796667233","https://openalex.org/W6798213610","https://openalex.org/W6801399157","https://openalex.org/W6802648193","https://openalex.org/W6803067813"],"related_works":["https://openalex.org/W3196155444","https://openalex.org/W3095538999","https://openalex.org/W3210156800","https://openalex.org/W3200361725","https://openalex.org/W4297094728","https://openalex.org/W3209574120","https://openalex.org/W2954428433","https://openalex.org/W3088796247","https://openalex.org/W3047894882","https://openalex.org/W2610686804"],"abstract_inverted_index":{"Unsupervised":[0],"pre-training":[1,34,98],"in":[2,17,73,186],"reinforcement":[3],"learning":[4,60,156],"enables":[5],"the":[6,18,29,49,108,117,121,161,187],"agent":[7,118],"to":[8,21,24,36,45,79,140,157],"gain":[9],"prior":[10],"environmental":[11],"knowledge,":[12],"which":[13],"is":[14,71,138],"then":[15],"fine-tuned":[16],"supervised":[19,62],"stage":[20],"quickly":[22],"adapt":[23],"various":[25],"downstream":[26],"tasks.":[27],"In":[28],"absence":[30],"of":[31,75,82,144,182],"task-related":[32],"rewards,":[33],"aims":[35],"acquire":[37],"policies":[38],"(i.e.,":[39],"behaviors)":[40],"that":[41,166],"generate":[42],"different":[43],"trajectories":[44],"explore":[46],"and":[47,87,116,129,147,173],"master":[48],"environment.":[50],"Previous":[51],"research":[52],"categorizes":[53],"states":[54,86,159],"into":[55,125,160],"their":[56],"associated":[57],"behaviors":[58],"by":[59],"a":[61,112,134,142,180],"discriminator.":[63],"However,":[64],"an":[65,80,95],"underlying":[66],"problem":[67],"persists:":[68],"such":[69],"discriminator":[70],"trained":[72],"lack":[74],"relevant":[76],"data,":[77],"leading":[78],"underestimation":[81],"reward":[83],"for":[84,100,127],"new":[85],"inadequate":[88],"exploration.":[89],"To":[90],"this":[91],"end,":[92],"we":[93,152],"introduce":[94],"unsupervised":[96,177],"active":[97],"algorithm":[99],"diverse":[101,130],"behavior":[102,109,131],"induction":[103],"(APD).":[104],"We":[105],"explicitly":[106],"characterize":[107],"variables":[110],"with":[111],"state-dependent":[113],"sampling":[114],"method,":[115],"can":[119,169],"decompose":[120],"entire":[122],"state":[123],"space":[124],"parts":[126],"fine-grained":[128],"learning.":[132],"Specifically,":[133],"particle-based":[135],"entropy":[136,146],"estimator":[137],"applied":[139],"optimize":[141],"combination":[143],"behavioral":[145],"mutual":[148],"information":[149],"objective.":[150],"Moreover,":[151],"develop":[153],"behavior-based":[154],"representation":[155],"compress":[158],"latent":[162],"space.":[163],"Experiments":[164],"show":[165],"our":[167],"method":[168],"improve":[170],"exploration":[171],"efficiency":[172],"outperforms":[174],"most":[175],"state-of-the-art":[176],"algorithms":[178],"on":[179],"number":[181],"continuous":[183],"control":[184],"tasks":[185],"DeepMind":[188],"Control":[189],"Suite.":[190]},"counts_by_year":[{"year":2024,"cited_by_count":3}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}