{"id":"https://openalex.org/W4308166345","doi":"https://doi.org/10.48550/arxiv.2211.00863","title":"Behavior Prior Representation learning for Offline Reinforcement Learning","display_name":"Behavior Prior Representation learning for Offline Reinforcement Learning","alternate_titles":[],"publication_year":2022,"publication_date":"2022-11-02","ids":{"openalex":"https://openalex.org/W4308166345","doi":"https://doi.org/10.48550/arxiv.2211.00863"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2211.00863","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.00863","pdf_url":"https://arxiv.org/pdf/2211.00863","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W4308166345"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2211.00863","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5031746269","display_name":"Hongyu Zang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zang, Hongyu","raw_affiliation_strings":["Beijing Institute of Technology , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology , China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100353701","display_name":"Xin Li","orcid":"https://orcid.org/0000-0001-6003-7719","observed_orcids":["https://orcid.org/0000-0001-6003-7719"]},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Li, Xin","raw_affiliation_strings":["Beijing Institute of Technology , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology , China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100409840","display_name":"Jie Yu","orcid":"https://orcid.org/0000-0002-5612-0103","observed_orcids":["https://orcid.org/0000-0002-5612-0103"]},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yu, Jie","raw_affiliation_strings":["Beijing Institute of Technology , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology , China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100322200","display_name":"Chen Liu","orcid":"https://orcid.org/0000-0003-1558-6836","observed_orcids":["https://orcid.org/0000-0003-1558-6836"]},"institutions":[{"id":"https://openalex.org/I125839683","display_name":"Beijing Institute of Technology","ror":"https://ror.org/01skt4w74","country_code":"CN","type":"education","lineage":["https://openalex.org/I125839683","https://openalex.org/I890469752"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Liu, Chen","raw_affiliation_strings":["Beijing Institute of Technology , China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Beijing Institute of Technology , China","institution_ids":["https://openalex.org/I125839683"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5079926596","display_name":"Riashat Islam","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Islam, Riashat","raw_affiliation_strings":["MILA , Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"MILA , Canada","institution_ids":["https://openalex.org/I4210164802"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080042640","display_name":"R\u00e9mi Tachet des Combes","orcid":"https://orcid.org/0000-0002-0763-8723","observed_orcids":["https://orcid.org/0000-0002-0763-8723"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Combes, Remi Tachet Des","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5089214987","display_name":"Romain Laroche","orcid":"https://orcid.org/0000-0001-7180-2746","observed_orcids":["https://orcid.org/0000-0001-7180-2746"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Laroche, Romain","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_pulled":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/offline-reinforcement-learning","display_name":"offline reinforcement learning","score":0.9929999709129333},{"id":"https://openalex.org/keywords/state-representation-learning","display_name":"state representation learning","score":0.8709999918937683},{"id":"https://openalex.org/keywords/behavioral-cloning","display_name":"behavior cloning","score":0.8790000081062317},{"id":"https://openalex.org/keywords/conservative-algorithms","display_name":"conservative algorithms","score":0.5329999923706055}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8861494064331055},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7680835127830505},{"id":"https://openalex.org/C29143872","wikidata":"https://www.wikidata.org/wiki/Q876690","display_name":"Business process reengineering","level":3,"score":0.6224550604820251},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5875458717346191},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5684988498687744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5641008019447327},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.5219499468803406},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.49970507621765137},{"id":"https://openalex.org/C2780102126","wikidata":"https://www.wikidata.org/wiki/Q10928179","display_name":"Online and offline","level":2,"score":0.42711177468299866},{"id":"https://openalex.org/C2780490138","wikidata":"https://www.wikidata.org/wiki/Q7079636","display_name":"Offline learning","level":3,"score":0.41777360439300537},{"id":"https://openalex.org/C9992130","wikidata":"https://www.wikidata.org/wiki/Q484954","display_name":"Pessimism","level":2,"score":0.4150979518890381},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.20578491687774658},{"id":"https://openalex.org/C2986087404","wikidata":"https://www.wikidata.org/wiki/Q15946010","display_name":"Online learning","level":2,"score":0.10731691122055054},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.0},{"id":"https://openalex.org/C21547014","wikidata":"https://www.wikidata.org/wiki/Q1423657","display_name":"Operations management","level":1,"score":0.0},{"id":"https://openalex.org/C136764020","wikidata":"https://www.wikidata.org/wiki/Q466","display_name":"World Wide Web","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C137335462","wikidata":"https://www.wikidata.org/wiki/Q380772","display_name":"Lean manufacturing","level":2,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2211.00863","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.00863","pdf_url":"https://arxiv.org/pdf/2211.00863","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2211.00863","doi":"https://doi.org/10.48550/arxiv.2211.00863","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2211.00863","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2211.00863","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2211.00863","pdf_url":"https://arxiv.org/pdf/2211.00863","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2796227307","display_name":null,"funder_award_id":"92270125","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G7144638067","display_name":null,"funder_award_id":"62276024","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320308943","display_name":"Microsoft Research","ror":"https://ror.org/00d0nc645"},{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4308166345.pdf","grobid_xml":"https://content.openalex.org/works/W4308166345.grobid-xml"},"referenced_works_count":43,"referenced_works":["https://openalex.org/W192920577","https://openalex.org/W1665214252","https://openalex.org/W2017779181","https://openalex.org/W2032378315","https://openalex.org/W2043804332","https://openalex.org/W2051228319","https://openalex.org/W2089561656","https://openalex.org/W2118022839","https://openalex.org/W2158782408","https://openalex.org/W2160808139","https://openalex.org/W2181962815","https://openalex.org/W2342840547","https://openalex.org/W2397240726","https://openalex.org/W2402540331","https://openalex.org/W2611328865","https://openalex.org/W2616345629","https://openalex.org/W2756826236","https://openalex.org/W2809005670","https://openalex.org/W2904730732","https://openalex.org/W2947150733","https://openalex.org/W2972435663","https://openalex.org/W2977481643","https://openalex.org/W3005727454","https://openalex.org/W3016525976","https://openalex.org/W3022566517","https://openalex.org/W3028766998","https://openalex.org/W3035060554","https://openalex.org/W3035180992","https://openalex.org/W3037625705","https://openalex.org/W3085605093","https://openalex.org/W3106426955","https://openalex.org/W3129266958","https://openalex.org/W3136208045","https://openalex.org/W3164527857","https://openalex.org/W3166052165","https://openalex.org/W3167222229","https://openalex.org/W3211479928","https://openalex.org/W3213037073","https://openalex.org/W3215067404","https://openalex.org/W4200634036","https://openalex.org/W4281626944","https://openalex.org/W4282813522","https://openalex.org/W4297808394"],"related_works":["https://openalex.org/W7134828545","https://openalex.org/W7117155180","https://openalex.org/W3126188503","https://openalex.org/W4362597907","https://openalex.org/W2991355586","https://openalex.org/W2995706821","https://openalex.org/W7108938610","https://openalex.org/W4409363334","https://openalex.org/W4415083830"],"abstract_inverted_index":{"Offline":[0,102,145],"reinforcement":[1],"learning":[2,53],"(RL)":[3],"struggles":[4],"in":[5],"environments":[6],"with":[7,65,142],"rich":[8],"and":[9,88],"noisy":[10],"inputs,":[11],"where":[12],"the":[13,33,74,86,96,131],"agent":[14],"only":[15],"has":[16],"access":[17],"to":[18,149],"a":[19,47,79,91],"fixed":[20,97],"dataset":[21],"without":[22],"environment":[23],"interactions.":[24],"Past":[25],"works":[26],"have":[27,119],"proposed":[28],"common":[29],"workarounds":[30],"based":[31,69],"on":[32,70,93],"pre-training":[34],"of":[35,73,95,130],"state":[36,54,63,80],"representations,":[37],"followed":[38],"by":[39,82],"policy":[40,92,121,132],"training.":[41],"In":[42],"this":[43],"work,":[44],"we":[45,76,106,137],"introduce":[46],"simple,":[48],"yet":[49],"effective":[50],"approach":[51],"for":[52],"representations.":[55],"Our":[56],"method,":[57],"Behavior":[58],"Prior":[59],"Representation":[60],"(BPR),":[61],"learns":[62],"representations":[64],"an":[66],"easy-to-integrate":[67],"objective":[68],"behavior":[71],"cloning":[72],"dataset:":[75],"first":[77],"learn":[78],"representation":[81],"mimicking":[83],"actions":[84],"from":[85],"dataset,":[87],"then":[89],"train":[90],"top":[94],"representation,":[98],"using":[99],"any":[100],"off-the-shelf":[101],"RL":[103,146],"algorithm.":[104],"Theoretically,":[105],"prove":[107],"that":[108,118,139],"BPR":[109,140],"carries":[110],"out":[111],"performance":[112],"guarantees":[113,123],"when":[114],"integrated":[115],"into":[116],"algorithms":[117,147],"either":[120],"improvement":[122],"(conservative":[124],"algorithms)":[125],"or":[126],"produce":[127],"lower":[128],"bounds":[129],"values":[133],"(pessimistic":[134],"algorithms).":[135],"Empirically,":[136],"show":[138],"combined":[141],"existing":[143],"state-of-the-art":[144],"leads":[148],"significant":[150],"improvements":[151],"across":[152],"several":[153],"offline":[154],"control":[155],"benchmarks.":[156],"The":[157],"code":[158],"is":[159],"available":[160],"at":[161],"\\url{https://github.com/bit1029public/offline_bpr}.":[162]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W4308166345","counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":3},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":2}],"updated_date":"2026-11-17T12:49:18.490984","created_date":"2022-11-08T00:00:00"}
