{"id":"https://openalex.org/W4393032096","doi":"https://doi.org/10.1109/tai.2024.3379969","title":"Enhancing Reinforcement Learning via Transformer-Based State Predictive Representations","display_name":"Enhancing Reinforcement Learning via Transformer-Based State Predictive Representations","publication_year":2024,"publication_date":"2024-03-21","ids":{"openalex":"https://openalex.org/W4393032096","doi":"https://doi.org/10.1109/tai.2024.3379969"},"language":"en","primary_location":{"id":"doi:10.1109/tai.2024.3379969","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2024.3379969","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5051746991","display_name":"Minsong Liu","orcid":"https://orcid.org/0009-0003-0822-1377"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Minsong Liu","raw_affiliation_strings":["National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0009-0003-0822-1377","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]},{"raw_affiliation_string":"National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080193690","display_name":"Yuanheng Zhu","orcid":"https://orcid.org/0000-0001-5384-423X"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yuanheng Zhu","raw_affiliation_strings":["National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-5384-423X","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]},{"raw_affiliation_string":"National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053853035","display_name":"Yaran Chen","orcid":"https://orcid.org/0000-0001-9356-0610"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Yaran Chen","raw_affiliation_strings":["National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-9356-0610","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100624298","display_name":"Dongbin Zhao","orcid":"https://orcid.org/0000-0001-8218-9633"},"institutions":[{"id":"https://openalex.org/I19820366","display_name":"Chinese Academy of Sciences","ror":"https://ror.org/034t30j35","country_code":"CN","type":"government","lineage":["https://openalex.org/I19820366"]},{"id":"https://openalex.org/I4210100255","display_name":"Beijing Academy of Artificial Intelligence","ror":"https://ror.org/016a74861","country_code":"CN","type":"other","lineage":["https://openalex.org/I4210100255"]},{"id":"https://openalex.org/I4210112150","display_name":"Institute of Automation","ror":"https://ror.org/022c3hy66","country_code":"CN","type":"facility","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"id":"https://openalex.org/I4210165038","display_name":"University of Chinese Academy of Sciences","ror":"https://ror.org/05qbk4x57","country_code":"CN","type":"education","lineage":["https://openalex.org/I19820366","https://openalex.org/I4210165038"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dongbin Zhao","raw_affiliation_strings":["National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China"],"raw_orcid":"https://orcid.org/0000-0001-8218-9633","affiliations":[{"raw_affiliation_string":"National Key Laboratory for Multimodal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]},{"raw_affiliation_string":"School of Artificial Intelligence, University of Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I4210100255","https://openalex.org/I4210165038"]},{"raw_affiliation_string":"National Key Laboratory for Multi-modal Artificial Intelligence Systems, Institute of Automation, Chinese Academy of Sciences, Beijing, China","institution_ids":["https://openalex.org/I19820366","https://openalex.org/I4210112150"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.1871,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.88431541,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":99},"biblio":{"volume":"5","issue":"9","first_page":"4364","last_page":"4375"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9997000098228455,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9794999957084656,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.978600025177002,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7813037633895874},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6799665689468384},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.6434192061424255},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5111027359962463},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.44245392084121704},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34474048018455505},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.23322266340255737},{"id":"https://openalex.org/keywords/electrical-engineering","display_name":"Electrical engineering","score":0.15823060274124146},{"id":"https://openalex.org/keywords/voltage","display_name":"Voltage","score":0.1397431492805481},{"id":"https://openalex.org/keywords/structural-engineering","display_name":"Structural engineering","score":0.08822768926620483}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7813037633895874},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6799665689468384},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.6434192061424255},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5111027359962463},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.44245392084121704},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34474048018455505},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.23322266340255737},{"id":"https://openalex.org/C119599485","wikidata":"https://www.wikidata.org/wiki/Q43035","display_name":"Electrical engineering","level":1,"score":0.15823060274124146},{"id":"https://openalex.org/C165801399","wikidata":"https://www.wikidata.org/wiki/Q25428","display_name":"Voltage","level":2,"score":0.1397431492805481},{"id":"https://openalex.org/C66938386","wikidata":"https://www.wikidata.org/wiki/Q633538","display_name":"Structural engineering","level":1,"score":0.08822768926620483}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tai.2024.3379969","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tai.2024.3379969","pdf_url":null,"source":{"id":"https://openalex.org/S4210169448","display_name":"IEEE Transactions on Artificial Intelligence","issn_l":"2691-4581","issn":["2691-4581"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Artificial Intelligence","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G7837090658","display_name":null,"funder_award_id":"4232056","funder_id":"https://openalex.org/F4320334977","funder_display_name":"Beijing Municipal Natural Science Foundation"},{"id":"https://openalex.org/G800914696","display_name":null,"funder_award_id":"62136008","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"},{"id":"https://openalex.org/G8179479325","display_name":null,"funder_award_id":"62293541","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"},{"id":"https://openalex.org/F4320334977","display_name":"Beijing Municipal Natural Science Foundation","ror":null}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":51,"referenced_works":["https://openalex.org/W2000474946","https://openalex.org/W2761873684","https://openalex.org/W2781585732","https://openalex.org/W2896457183","https://openalex.org/W2904246096","https://openalex.org/W2966477753","https://openalex.org/W3005173720","https://openalex.org/W3084241738","https://openalex.org/W3093210455","https://openalex.org/W3093528669","https://openalex.org/W3110979110","https://openalex.org/W3127235394","https://openalex.org/W3130635520","https://openalex.org/W3169924623","https://openalex.org/W3175558129","https://openalex.org/W3205321526","https://openalex.org/W4252279978","https://openalex.org/W4307774253","https://openalex.org/W4312335991","https://openalex.org/W4313156423","https://openalex.org/W4313185587","https://openalex.org/W4366493064","https://openalex.org/W4367663369","https://openalex.org/W4379740312","https://openalex.org/W4385245566","https://openalex.org/W4385413695","https://openalex.org/W4386470330","https://openalex.org/W6747387971","https://openalex.org/W6755207826","https://openalex.org/W6757592117","https://openalex.org/W6760405395","https://openalex.org/W6765407481","https://openalex.org/W6768810269","https://openalex.org/W6768843968","https://openalex.org/W6771217966","https://openalex.org/W6771570156","https://openalex.org/W6773738434","https://openalex.org/W6776867236","https://openalex.org/W6776901495","https://openalex.org/W6780470247","https://openalex.org/W6782858274","https://openalex.org/W6787959036","https://openalex.org/W6790416196","https://openalex.org/W6791194670","https://openalex.org/W6796848067","https://openalex.org/W6800217721","https://openalex.org/W6800570120","https://openalex.org/W6804859080","https://openalex.org/W6810343060","https://openalex.org/W6838121908","https://openalex.org/W6844194202"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W3046775127","https://openalex.org/W3107602296","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W4364306694","https://openalex.org/W4312192474","https://openalex.org/W4283697347"],"abstract_inverted_index":{"Enhancing":[0],"state":[1,30,60,108,125],"representations":[2,31,62,109],"can":[3],"effectively":[4,115],"mitigate":[5],"the":[6,111,130,141,155,169],"issue":[7],"of":[8,48,132,143,158],"low":[9],"sample":[10,25,156],"efficiency":[11,26,157],"in":[12,41,135,154],"reinforcement":[13],"learning":[14,28,47,88],"(RL)":[15],"within":[16,50,110,119],"high-dimensional":[17],"input":[18],"environments.":[19],"Existing":[20],"methods":[21,166],"attempt":[22],"to":[23,99,121,129,150],"improve":[24],"by":[27,54],"predictive":[29,61,97],"from":[32],"sequence":[33],"data.":[34],"However,":[35],"there":[36],"still":[37],"remain":[38],"significant":[39],"challenges":[40],"achieving":[42],"a":[43,58,95,151],"comprehensive":[44],"understanding":[45],"and":[46,102,174],"information":[49,118],"long":[51],"sequences.":[52],"Motivated":[53],"this,":[55],"we":[56,93],"introduce":[57],"transformer-based":[59,96],"(TSPR":[63],"<xref":[64],"ref-type=\"fn\"":[65],"rid=\"fn1\"":[66],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"":[67,71],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><sup>1</sup></xref>":[68],"<fn":[69],"id=\"fn1\"":[70],"xmlns:xlink=\"http://www.w3.org/1999/xlink\"><label><sup>1</sup></label>":[72],"Our":[73],"code":[74],"will":[75],"be":[76],"released":[77],"at":[78],"<uri>https://github.com/gourmet-liu/TSPR</uri>":[79],"</fn>":[80],")":[81],"auxiliary":[82],"task":[83],"that":[84,140],"promotes":[85],"better":[86],"representation":[87],"through":[89],"self-supervised":[90],"goals.":[91],"Specifically,":[92],"design":[94],"model":[98],"establish":[100],"unidirectional":[101],"bidirectional":[103],"prediction":[104],"tasks":[105],"for":[106],"predicting":[107],"latent":[112],"space.":[113],"TSPR":[114,144,161],"exploits":[116],"contextual":[117],"sequences":[120],"learn":[122],"more":[123],"informative":[124],"representations,":[126],"thereby":[127],"contributing":[128],"enhancement":[131],"policy":[133],"training":[134],"RL.":[136,159],"Extensive":[137],"experiments":[138],"demonstrate":[139],"combination":[142],"with":[145],"off-policy":[146],"RL":[147,165],"algorithms":[148],"leads":[149],"substantial":[152],"improvement":[153],"Furthermore,":[160],"outperforms":[162],"state-of-the-art":[163],"sample-efficient":[164],"on":[167],"both":[168],"multiple":[170],"continuous":[171],"control":[172,176],"(DMControl)":[173],"discrete":[175],"(Atari)":[177],"tasks.":[178]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":6},{"year":2024,"cited_by_count":2}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}