{"id":"https://openalex.org/W3207337643","doi":"https://doi.org/10.48550/arxiv.2110.03743","title":"Reinforcement Learning in Reward-Mixing MDPs","display_name":"Reinforcement Learning in Reward-Mixing MDPs","publication_year":2021,"publication_date":"2021-10-07","ids":{"openalex":"https://openalex.org/W3207337643","doi":"https://doi.org/10.48550/arxiv.2110.03743","mag":"3207337643"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2110.03743","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.03743","pdf_url":"https://arxiv.org/pdf/2110.03743","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2110.03743","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5000527109","display_name":"Jeongyeol Kwon","orcid":"https://orcid.org/0000-0002-7910-7817","observed_orcids":["https://orcid.org/0000-0002-7910-7817"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kwon, Jeongyeol","raw_affiliation_strings":["The University of Texas at Austin","University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090891199","display_name":"Yonathan Efroni","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1290206253","display_name":"Microsoft (United States)","ror":"https://ror.org/00d0nc645","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253"]},{"id":"https://openalex.org/I4401726785","display_name":"Microsoft Research New York City (United States)","ror":"https://ror.org/056zprp28","country_code":"US","type":"company","lineage":["https://openalex.org/I1290206253","https://openalex.org/I4401726785"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Efroni, Yonathan","raw_affiliation_strings":["Microsoft Research , NYC","Microsoft (United States), Redmond, United States"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Microsoft Research , NYC","institution_ids":["https://openalex.org/I4401726785"]},{"raw_affiliation_string":"Microsoft (United States), Redmond, United States","institution_ids":["https://openalex.org/I1290206253"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053978837","display_name":"Constantine Caramanis","orcid":"https://orcid.org/0000-0001-9939-8378","observed_orcids":["https://orcid.org/0000-0001-9939-8378"]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Caramanis, Constantine","raw_affiliation_strings":["The University of Texas at Austin","University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]},{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5036260775","display_name":"Shie Mannor","orcid":"https://orcid.org/0000-0003-4439-7647","observed_orcids":["https://orcid.org/0000-0003-4439-7647"]},"institutions":[{"id":"https://openalex.org/I174306211","display_name":"Technion \u2013 Israel Institute of Technology","ror":"https://ror.org/03qryx823","country_code":"IL","type":"education","lineage":["https://openalex.org/I174306211"]}],"countries":["IL"],"is_corresponding":false,"raw_author_name":"Mannor, Shie","raw_affiliation_strings":["Technion ,","Technion-Israel Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Technion ,","institution_ids":["https://openalex.org/I174306211"]},{"raw_affiliation_string":"Technion-Israel Institute of Technology","institution_ids":["https://openalex.org/I174306211"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":3,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"34","issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9990000128746033,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.000699999975040555,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/partially-observable-markov-decision-processes","display_name":"partially observable Markov decision processes","score":0.8379999995231628},{"id":"https://openalex.org/keywords/episodic-reinforcement-learning","display_name":"episodic reinforcement learning","score":0.949999988079071},{"id":"https://openalex.org/keywords/optimal-policy-learning","display_name":"optimal policy learning","score":0.6050000190734863},{"id":"https://openalex.org/keywords/polynomial-time-algorithms","display_name":"polynomial-time algorithms","score":0.7310000061988831},{"id":"https://openalex.org/keywords/latent-state-space","display_name":"latent state space","score":0.6710000038146973}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8872542381286621},{"id":"https://openalex.org/C2780767217","wikidata":"https://www.wikidata.org/wiki/Q5532421","display_name":"Generality","level":2,"score":0.7674341201782227},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.7072520852088928},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.6470051407814026},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.5981881618499756},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5445618033409119},{"id":"https://openalex.org/C2778355321","wikidata":"https://www.wikidata.org/wiki/Q17079427","display_name":"Identity (music)","level":2,"score":0.49288153648376465},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.48552775382995605},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.474423348903656},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.4631824493408203},{"id":"https://openalex.org/C32848918","wikidata":"https://www.wikidata.org/wiki/Q845789","display_name":"Observable","level":2,"score":0.4487423896789551},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.4410611689090729},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4140247106552124},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.41078996658325195},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.33443403244018555},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3240293860435486},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2178489863872528},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.15234971046447754},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.12454473972320557},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07706108689308167},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0},{"id":"https://openalex.org/C24890656","wikidata":"https://www.wikidata.org/wiki/Q82811","display_name":"Acoustics","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"pmh:oai:arXiv.org:2110.03743","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.03743","pdf_url":"https://arxiv.org/pdf/2110.03743","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3207337643","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2110.03743.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2110.03743","doi":"https://doi.org/10.48550/arxiv.2110.03743","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2110.03743","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},{"id":"mag:3212134242","doi":null,"is_oa":false,"landing_page_url":"https://dblp.uni-trier.de/db/journals/corr/corr2110.html#abs-2110-03743","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Neural Information Processing Systems","raw_type":null}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2110.03743","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2110.03743","pdf_url":"https://arxiv.org/pdf/2110.03743","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G3024042299","display_name":null,"funder_award_id":"W911NF19-2-0333","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G608057831","display_name":null,"funder_award_id":"W911NF-19","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7452299184","display_name":null,"funder_award_id":"W911NF","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G8415477955","display_name":null,"funder_award_id":"W911NF-19","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G8429103732","display_name":"AI Institute: Institute for Foundations of Machine Learning","funder_award_id":"2019844","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320322252","display_name":"Israel Science Foundation","ror":"https://ror.org/04sazxf24"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":45,"referenced_works":["https://openalex.org/W195596278","https://openalex.org/W201409579","https://openalex.org/W1517383877","https://openalex.org/W1763164889","https://openalex.org/W1825640910","https://openalex.org/W1850488217","https://openalex.org/W1956647075","https://openalex.org/W1974648697","https://openalex.org/W1976238508","https://openalex.org/W2031823362","https://openalex.org/W2103708221","https://openalex.org/W2110962519","https://openalex.org/W2121863487","https://openalex.org/W2125710232","https://openalex.org/W2156211713","https://openalex.org/W2159392853","https://openalex.org/W2163840227","https://openalex.org/W2241126168","https://openalex.org/W2273762088","https://openalex.org/W2295214655","https://openalex.org/W2470693974","https://openalex.org/W2489939061","https://openalex.org/W2545659366","https://openalex.org/W2569188995","https://openalex.org/W2768353504","https://openalex.org/W2898656812","https://openalex.org/W2912099989","https://openalex.org/W2912399346","https://openalex.org/W2942689850","https://openalex.org/W2947782105","https://openalex.org/W2962723383","https://openalex.org/W2962956149","https://openalex.org/W2963971282","https://openalex.org/W2964054583","https://openalex.org/W2970870329","https://openalex.org/W2971249033","https://openalex.org/W2991935368","https://openalex.org/W3004977066","https://openalex.org/W3034548573","https://openalex.org/W3046395471","https://openalex.org/W3101472277","https://openalex.org/W3118621910","https://openalex.org/W3134141736","https://openalex.org/W3139488757","https://openalex.org/W3213247829"],"related_works":["https://openalex.org/W4303648811","https://openalex.org/W4415796570","https://openalex.org/W7214948301","https://openalex.org/W2377907131","https://openalex.org/W3202898482","https://openalex.org/W4327993687","https://openalex.org/W3212671443","https://openalex.org/W2273762088","https://openalex.org/W3213247829","https://openalex.org/W7103455158"],"abstract_inverted_index":{"Learning":[0],"a":[1,6,27,34,89],"near":[2,90],"optimal":[3,91],"policy":[4,92,154],"in":[5,14,26,116,189],"partially":[7,190],"observable":[8],"system":[9],"remains":[10],"an":[11,152],"elusive":[12],"challenge":[13],"contemporary":[15],"reinforcement":[16,24],"learning.":[17],"In":[18],"this":[19],"work,":[20],"we":[21,108],"consider":[22],"episodic":[23],"learning":[25,88],"reward-mixing":[28,95],"Markov":[29],"decision":[30],"process":[31],"(MDP).":[32],"There,":[33],"reward":[35,44,58],"function":[36],"is":[37,60,77,164,178,197],"drawn":[38],"from":[39],"one":[40],"of":[41,49,55,87,172],"multiple":[42],"possible":[43],"models":[45],"at":[46],"the":[47,53,56,64,67,73,81,85,106,114,129,146,170,179,194,200],"beginning":[48],"every":[50],"episode,":[51],"but":[52],"identity":[54],"chosen":[57],"model":[59],"not":[61,78,185],"revealed":[62],"to":[63,80],"agent.":[65,82],"Hence,":[66],"latent":[68,201],"state":[69,202],"space,":[70],"for":[71,93,125,141],"which":[72],"dynamics":[74],"are":[75,169],"Markovian,":[76],"given":[79],"We":[83,144],"study":[84,113],"problem":[86,115,130],"two":[94,126],"MDPs.":[96],"Unlike":[97],"existing":[98,136],"approaches":[99],"that":[100,150,183],"rely":[101],"on":[102,105],"strong":[103],"assumptions":[104,111,188],"dynamics,":[107],"make":[109],"no":[110,121],"and":[112,138,166,174],"full":[117],"generality.":[118],"Indeed,":[119],"with":[120],"further":[122],"assumptions,":[123],"even":[124],"switching":[127],"reward-models,":[128],"requires":[131],"several":[132],"new":[133],"ideas":[134],"beyond":[135],"algorithmic":[137],"analysis":[139],"techniques":[140],"efficient":[142,181],"exploration.":[143],"provide":[145],"first":[147,180],"polynomial-time":[148],"algorithm":[149,182],"finds":[151],"$\u03b5$-optimal":[153],"after":[155],"exploring":[156],"$\\tilde{O}(poly(H,\u03b5^{-1})":[157],"\\cdot":[158],"S^2":[159],"A^2)$":[160],"episodes,":[161],"where":[162,193],"$H$":[163],"time-horizon":[165],"$S,":[167],"A$":[168],"number":[171],"states":[173],"actions":[175],"respectively.":[176],"This":[177],"does":[184],"require":[186],"any":[187],"observed":[191],"environments":[192],"observation":[195],"space":[196],"smaller":[198],"than":[199],"space.":[203]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":1}],"updated_date":"2026-10-27T15:51:10.671819","created_date":"2021-10-25T00:00:00"}