{"id":"https://openalex.org/W4393305228","doi":"https://doi.org/10.1109/jsait.2024.3381818","title":"Toward General Function Approximation in Nonstationary Reinforcement Learning","display_name":"Toward General Function Approximation in Nonstationary Reinforcement Learning","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4393305228","doi":"https://doi.org/10.1109/jsait.2024.3381818"},"language":"en","primary_location":{"id":"doi:10.1109/jsait.2024.3381818","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsait.2024.3381818","pdf_url":null,"source":{"id":"https://openalex.org/S4210211895","display_name":"IEEE Journal on Selected Areas in Information Theory","issn_l":"2641-8770","issn":["2641-8770"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Information Theory","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057599205","display_name":"Songtao Feng","orcid":"https://orcid.org/0000-0002-7773-5406"},"institutions":[{"id":"https://openalex.org/I33213144","display_name":"University of Florida","ror":"https://ror.org/02y3ad647","country_code":"US","type":"education","lineage":["https://openalex.org/I33213144"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Songtao Feng","raw_affiliation_strings":["Department of Electrical and Computer Engineering, University of Florida, Gainesville, FL, USA"],"raw_orcid":"https://orcid.org/0000-0001-9062-3110","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, University of Florida, Gainesville, FL, USA","institution_ids":["https://openalex.org/I33213144"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101993864","display_name":"Ming Yin","orcid":"https://orcid.org/0000-0002-7037-1048"},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ming Yin","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Princeton University, Princeton, NJ, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Princeton University, Princeton, NJ, USA","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5025491239","display_name":"Ruiquan Huang","orcid":null},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ruiquan Huang","raw_affiliation_strings":["School of Electrical Engineering and Computer Science, The Pennsylvania State University, University Park, PA, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, The Pennsylvania State University, University Park, PA, USA","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100394392","display_name":"Yu-Xiang Wang","orcid":"https://orcid.org/0000-0002-6403-212X"},"institutions":[{"id":"https://openalex.org/I154570441","display_name":"University of California, Santa Barbara","ror":"https://ror.org/02t274463","country_code":"US","type":"education","lineage":["https://openalex.org/I154570441"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yu-Xiang Wang","raw_affiliation_strings":["Department of Computer Science, University of California at Santa Barbara, Santa Barbara, CA, USA"],"raw_orcid":"https://orcid.org/0000-0002-6403-212X","affiliations":[{"raw_affiliation_string":"Department of Computer Science, University of California at Santa Barbara, Santa Barbara, CA, USA","institution_ids":["https://openalex.org/I154570441"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100645589","display_name":"Jing Yang","orcid":"https://orcid.org/0000-0002-6009-864X"},"institutions":[{"id":"https://openalex.org/I130769515","display_name":"Pennsylvania State University","ror":"https://ror.org/04p491231","country_code":"US","type":"education","lineage":["https://openalex.org/I130769515"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Jing Yang","raw_affiliation_strings":["School of Electrical Engineering and Computer Science, The Pennsylvania State University, University Park, PA, USA"],"raw_orcid":"https://orcid.org/0000-0002-6009-864X","affiliations":[{"raw_affiliation_string":"School of Electrical Engineering and Computer Science, The Pennsylvania State University, University Park, PA, USA","institution_ids":["https://openalex.org/I130769515"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100384384","display_name":"Yingbin Liang","orcid":"https://orcid.org/0000-0003-2631-4262"},"institutions":[{"id":"https://openalex.org/I52357470","display_name":"The Ohio State University","ror":"https://ror.org/00rs6vg23","country_code":"US","type":"education","lineage":["https://openalex.org/I52357470"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yingbin Liang","raw_affiliation_strings":["Department of Electrical and Computer Engineering, The Ohio State University, Columbus, OH, USA"],"raw_orcid":"https://orcid.org/0000-0003-2631-4262","affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, The Ohio State University, Columbus, OH, USA","institution_ids":["https://openalex.org/I52357470"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":6,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.3055,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.60265001,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":91,"max":95},"biblio":{"volume":"5","issue":null,"first_page":"190","last_page":"206"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9650999903678894,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9650999903678894,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6992233395576477},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.5336469411849976},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5171029567718506},{"id":"https://openalex.org/keywords/function-approximation","display_name":"Function approximation","score":0.44932302832603455},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3865357041358948},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.28656020760536194},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.2143358290195465},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.11837151646614075},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.08387154340744019},{"id":"https://openalex.org/keywords/biology","display_name":"Biology","score":0.06386414170265198}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6992233395576477},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.5336469411849976},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5171029567718506},{"id":"https://openalex.org/C91873725","wikidata":"https://www.wikidata.org/wiki/Q3445816","display_name":"Function approximation","level":3,"score":0.44932302832603455},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3865357041358948},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.28656020760536194},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2143358290195465},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.11837151646614075},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.08387154340744019},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.06386414170265198},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/jsait.2024.3381818","is_oa":false,"landing_page_url":"https://doi.org/10.1109/jsait.2024.3381818","pdf_url":null,"source":{"id":"https://openalex.org/S4210211895","display_name":"IEEE Journal on Selected Areas in Information Theory","issn_l":"2641-8770","issn":["2641-8770"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Journal on Selected Areas in Information Theory","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Reduced inequalities","score":0.41999998688697815,"id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G1638887437","display_name":null,"funder_award_id":"CNS-1956276","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2364766549","display_name":null,"funder_award_id":"CNS-2003131","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3755522344","display_name":null,"funder_award_id":"CNS-2030026","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":58,"referenced_works":["https://openalex.org/W157259654","https://openalex.org/W1977655452","https://openalex.org/W1981773323","https://openalex.org/W2074680702","https://openalex.org/W2104753538","https://openalex.org/W2151161180","https://openalex.org/W2156211713","https://openalex.org/W2158130000","https://openalex.org/W2162926979","https://openalex.org/W2562337727","https://openalex.org/W2575705757","https://openalex.org/W2772709170","https://openalex.org/W2902907165","https://openalex.org/W2949910245","https://openalex.org/W2953303875","https://openalex.org/W2953305686","https://openalex.org/W2982316857","https://openalex.org/W2987195113","https://openalex.org/W2998187693","https://openalex.org/W3091875946","https://openalex.org/W3127248851","https://openalex.org/W3206776381","https://openalex.org/W4225838216","https://openalex.org/W4247692611","https://openalex.org/W4283711528","https://openalex.org/W4287633962","https://openalex.org/W4295177675","https://openalex.org/W4300980646","https://openalex.org/W6604079820","https://openalex.org/W6675595744","https://openalex.org/W6679661297","https://openalex.org/W6681980492","https://openalex.org/W6683896386","https://openalex.org/W6684050718","https://openalex.org/W6690381995","https://openalex.org/W6729229454","https://openalex.org/W6746177919","https://openalex.org/W6751346906","https://openalex.org/W6758580343","https://openalex.org/W6761973403","https://openalex.org/W6762889291","https://openalex.org/W6779771741","https://openalex.org/W6779775941","https://openalex.org/W6780200251","https://openalex.org/W6780394777","https://openalex.org/W6780779400","https://openalex.org/W6784134107","https://openalex.org/W6784344538","https://openalex.org/W6790179192","https://openalex.org/W6790330943","https://openalex.org/W6791269304","https://openalex.org/W6797339219","https://openalex.org/W6803178339","https://openalex.org/W6810918382","https://openalex.org/W6838630712","https://openalex.org/W6844927581","https://openalex.org/W6853324332","https://openalex.org/W6922480057"],"related_works":["https://openalex.org/W2748952813","https://openalex.org/W2920061524","https://openalex.org/W4310083477","https://openalex.org/W2328553770","https://openalex.org/W1977959518","https://openalex.org/W2038908348","https://openalex.org/W2107890255","https://openalex.org/W3021988786","https://openalex.org/W3099311996","https://openalex.org/W2963971282"],"abstract_inverted_index":{"Function":[0],"approximation":[1,26,38],"has":[2],"experienced":[3],"significant":[4],"success":[5],"in":[6,162,186],"the":[7,64,90,94,108,117,124,129,134,142,153,163,169,174,178,187],"field":[8],"of":[9,16,128,141,171,189],"reinforcement":[10],"learning":[11],"(RL).":[12],"Despite":[13],"a":[14,69,84,99,113,138],"handful":[15],"progress":[17],"on":[18,89],"developing":[19],"theory":[20],"for":[21,32,50,78,104,156],"nonstationary":[22,33,51,79,105,157,190],"RL":[23,34,52],"with":[24,35,53],"function":[25,37,55],"under":[27],"structural":[28],"assumptions,":[29],"existing":[30,154],"work":[31],"general":[36,54],"is":[39],"still":[40],"limited.":[41],"In":[42],"this":[43],"work,":[44],"we":[45,67,111],"investigate":[46],"two":[47,149,175],"different":[48],"approaches":[49,176],"approximation:":[56],"confidence-set":[57,85,130,180],"based":[58,86,88,131,181],"algorithm":[59,87,115,182,185],"and":[60,81,98,122,137,159,183],"UCB-type":[61,114,184],"algorithm.":[62],"For":[63,107],"first":[65,179],"approach,":[66,110],"introduce":[68],"new":[70,139],"complexity":[71,91],"measure":[72],"called":[73],"dynamic":[74],"Bellman":[75],"Eluder":[76],"(DBE)":[77],"MDPs,":[80],"then":[82],"propose":[83,112],"metric.":[92],"features":[93,133],"sliding":[95],"window":[96],"mechanism":[97,136],"novel":[100],"confidence":[101],"set":[102],"design":[103,140],"MDPs.":[106,191],"second":[109],"following":[116],"popular":[118],"least-square-value-iteration":[119],"(LSVI)":[120],"framework,":[121],"mitigate":[123],"computational":[125],"efficiency":[126],"challenge":[127],"approach.":[132],"restart":[135],"bonus":[143],"term":[144],"to":[145],"handle":[146],"nonstationarity.":[147],"The":[148],"proposed":[150],"algorithms":[151,155],"outperform":[152],"linear":[158],"tabular":[160],"MDPs":[161],"small":[164],"variation":[165],"budget":[166],"setting.":[167],"To":[168],"best":[170],"our":[172],"knowledge,":[173],"are":[177],"context":[188]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2024-03-30T00:00:00"}