{"id":"https://openalex.org/W2581566809","doi":"https://doi.org/10.1109/tac.2018.2866455","title":"Learning Policies for Markov Decision Processes From Data","display_name":"Learning Policies for Markov Decision Processes From Data","publication_year":2018,"publication_date":"2018-08-21","ids":{"openalex":"https://openalex.org/W2581566809","doi":"https://doi.org/10.1109/tac.2018.2866455","mag":"2581566809"},"language":"en","primary_location":{"id":"doi:10.1109/tac.2018.2866455","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tac.2018.2866455","pdf_url":null,"source":{"id":"https://openalex.org/S184954342","display_name":"IEEE Transactions on Automatic Control","issn_l":"0018-9286","issn":["0018-9286","1558-2523","2334-3303"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automatic Control","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057858044","display_name":"Manjesh K. Hanawal","orcid":"https://orcid.org/0000-0002-1807-5487"},"institutions":[{"id":"https://openalex.org/I162827531","display_name":"Indian Institute of Technology Bombay","ror":"https://ror.org/02qyf5152","country_code":"IN","type":"education","lineage":["https://openalex.org/I162827531"]}],"countries":["IN"],"is_corresponding":true,"raw_author_name":"Manjesh Kumar Hanawal","raw_affiliation_strings":["Industrial Engineering and Operations Research, Indian Institute of Bombay, Mumbai, India"],"affiliations":[{"raw_affiliation_string":"Industrial Engineering and Operations Research, Indian Institute of Bombay, Mumbai, India","institution_ids":["https://openalex.org/I162827531"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100458867","display_name":"Hao Liu","orcid":"https://orcid.org/0000-0003-0510-6501"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Hao Liu","raw_affiliation_strings":["College of Control Science and Engineering, Zhejiang University, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"College of Control Science and Engineering, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5054049845","display_name":"Henghui Zhu","orcid":"https://orcid.org/0000-0002-4534-6975"},"institutions":[{"id":"https://openalex.org/I76130692","display_name":"Zhejiang University","ror":"https://ror.org/00a2xv884","country_code":"CN","type":"education","lineage":["https://openalex.org/I76130692"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Henghui Zhu","raw_affiliation_strings":["College of Control Science and Engineering, Zhejiang University, Hangzhou, China"],"affiliations":[{"raw_affiliation_string":"College of Control Science and Engineering, Zhejiang University, Hangzhou, China","institution_ids":["https://openalex.org/I76130692"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5075696701","display_name":"Ioannis Ch. Paschalidis","orcid":"https://orcid.org/0000-0002-3343-2913"},"institutions":[{"id":"https://openalex.org/I111088046","display_name":"Boston University","ror":"https://ror.org/05qwgg493","country_code":"US","type":"education","lineage":["https://openalex.org/I111088046"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Ioannis Ch. Paschalidis","raw_affiliation_strings":["Boston University, Boston, MA, US"],"affiliations":[{"raw_affiliation_string":"Boston University, Boston, MA, US","institution_ids":["https://openalex.org/I111088046"]}]}],"institutions":[],"countries_distinct_count":3,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5057858044"],"corresponding_institution_ids":["https://openalex.org/I162827531"],"apc_list":null,"apc_paid":null,"fwci":1.466,"has_fulltext":false,"cited_by_count":17,"citation_normalized_percentile":{"value":0.86011534,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":98},"biblio":{"volume":"64","issue":"6","first_page":"2298","last_page":"2309"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.998199999332428,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9919000267982483,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.6414310932159424},{"id":"https://openalex.org/keywords/regret","display_name":"Regret","score":0.632615327835083},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.6224644780158997},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.5221662521362305},{"id":"https://openalex.org/keywords/reproducing-kernel-hilbert-space","display_name":"Reproducing kernel Hilbert space","score":0.5126554369926453},{"id":"https://openalex.org/keywords/markov-kernel","display_name":"Markov kernel","score":0.49849796295166016},{"id":"https://openalex.org/keywords/partially-observable-markov-decision-process","display_name":"Partially observable Markov decision process","score":0.4910736083984375},{"id":"https://openalex.org/keywords/kernel","display_name":"Kernel (algebra)","score":0.4685087502002716},{"id":"https://openalex.org/keywords/kernel-regression","display_name":"Kernel regression","score":0.46004346013069153},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.44390669465065},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4405458867549896},{"id":"https://openalex.org/keywords/state-space","display_name":"State space","score":0.42213818430900574},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.4102402627468109},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.40348124504089355},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.37533992528915405},{"id":"https://openalex.org/keywords/markov-model","display_name":"Markov model","score":0.3732750415802002},{"id":"https://openalex.org/keywords/variable-order-markov-model","display_name":"Variable-order Markov model","score":0.37307649850845337},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.32483184337615967},{"id":"https://openalex.org/keywords/hilbert-space","display_name":"Hilbert space","score":0.26342320442199707},{"id":"https://openalex.org/keywords/econometrics","display_name":"Econometrics","score":0.26238441467285156},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.25899550318717957},{"id":"https://openalex.org/keywords/nonparametric-statistics","display_name":"Nonparametric statistics","score":0.17317837476730347},{"id":"https://openalex.org/keywords/discrete-mathematics","display_name":"Discrete mathematics","score":0.1389407217502594}],"concepts":[{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.6414310932159424},{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.632615327835083},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.6224644780158997},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5221662521362305},{"id":"https://openalex.org/C80884492","wikidata":"https://www.wikidata.org/wiki/Q3345678","display_name":"Reproducing kernel Hilbert space","level":3,"score":0.5126554369926453},{"id":"https://openalex.org/C106666656","wikidata":"https://www.wikidata.org/wiki/Q6771323","display_name":"Markov kernel","level":5,"score":0.49849796295166016},{"id":"https://openalex.org/C17098449","wikidata":"https://www.wikidata.org/wiki/Q176814","display_name":"Partially observable Markov decision process","level":4,"score":0.4910736083984375},{"id":"https://openalex.org/C74193536","wikidata":"https://www.wikidata.org/wiki/Q574844","display_name":"Kernel (algebra)","level":2,"score":0.4685087502002716},{"id":"https://openalex.org/C200695384","wikidata":"https://www.wikidata.org/wiki/Q1739319","display_name":"Kernel regression","level":3,"score":0.46004346013069153},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.44390669465065},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4405458867549896},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.42213818430900574},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.4102402627468109},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.40348124504089355},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.37533992528915405},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.3732750415802002},{"id":"https://openalex.org/C54907487","wikidata":"https://www.wikidata.org/wiki/Q7915688","display_name":"Variable-order Markov model","level":4,"score":0.37307649850845337},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.32483184337615967},{"id":"https://openalex.org/C62799726","wikidata":"https://www.wikidata.org/wiki/Q190056","display_name":"Hilbert space","level":2,"score":0.26342320442199707},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.26238441467285156},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.25899550318717957},{"id":"https://openalex.org/C102366305","wikidata":"https://www.wikidata.org/wiki/Q1097688","display_name":"Nonparametric statistics","level":2,"score":0.17317837476730347},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.1389407217502594},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tac.2018.2866455","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tac.2018.2866455","pdf_url":null,"source":{"id":"https://openalex.org/S184954342","display_name":"IEEE Transactions on Automatic Control","issn_l":"0018-9286","issn":["0018-9286","1558-2523","2334-3303"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automatic Control","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.8100000023841858,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G2111622434","display_name":null,"funder_award_id":"CNS-1645681","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G2782532677","display_name":null,"funder_award_id":"DMS-1664644","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5031208370","display_name":null,"funder_award_id":"W911NF-12-1-0390","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G5821627941","display_name":null,"funder_award_id":"IIS-1237022","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8379567014","display_name":null,"funder_award_id":"MURI N00014-16-1-2832","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G960070933","display_name":null,"funder_award_id":"CCF-1527292","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":31,"referenced_works":["https://openalex.org/W60181528","https://openalex.org/W215298514","https://openalex.org/W1521625446","https://openalex.org/W1542886316","https://openalex.org/W1931877416","https://openalex.org/W1964782533","https://openalex.org/W1971547063","https://openalex.org/W1983161979","https://openalex.org/W1986014385","https://openalex.org/W2009303086","https://openalex.org/W2059486486","https://openalex.org/W2098432798","https://openalex.org/W2114938622","https://openalex.org/W2133853511","https://openalex.org/W2162870748","https://openalex.org/W2164542992","https://openalex.org/W2166302491","https://openalex.org/W2326968954","https://openalex.org/W2334782222","https://openalex.org/W2400458653","https://openalex.org/W2478708596","https://openalex.org/W2487144912","https://openalex.org/W2531781944","https://openalex.org/W2571465252","https://openalex.org/W2950989964","https://openalex.org/W4292025355","https://openalex.org/W6602468054","https://openalex.org/W6608635065","https://openalex.org/W6640174482","https://openalex.org/W6677631742","https://openalex.org/W7046297837"],"related_works":["https://openalex.org/W3124083798","https://openalex.org/W1649547938","https://openalex.org/W2112116699","https://openalex.org/W2017332415","https://openalex.org/W2157199237","https://openalex.org/W3102128462","https://openalex.org/W1985664346","https://openalex.org/W3023188449","https://openalex.org/W2063024474","https://openalex.org/W3104611551"],"abstract_inverted_index":{"We":[0,25,93,178],"consider":[1],"the":[2,18,23,27,33,55,79,97,100,104,107,115,119,123,139,163,173,176,180],"problem":[3],"of":[4,57,83,103,114,122,138,142,162,165,175,182],"learning":[5],"a":[6,9,47,64,75,85,186],"policy":[7,28,77,86],"for":[8],"Markov":[10,125,143],"decision":[11],"process":[12],"consistent":[13],"with":[14,32],"data":[15],"captured":[16],"on":[17,96,185],"state-action":[19,34],"pairs":[20],"followed":[21],"by":[22],"policy.":[24],"parameterize":[26],"using":[29,43,89],"features":[30,37,58],"associated":[31],"pairs.":[35],"The":[36,81],"can":[38,59],"be":[39,60,71],"handcrafted":[40],"or":[41],"defined":[42],"kernel":[44,49],"functions":[45],"in":[46,112],"reproducing":[48],"Hilbert":[50],"space.":[51],"In":[52],"either":[53],"case,":[54],"set":[56],"large":[61],"and":[62,106,118,135],"only":[63],"small,":[65],"unknown":[66,108],"subset":[67],"may":[68],"need":[69],"to":[70,73,78,149,157],"used":[72],"fit":[74],"specific":[76],"data.":[80],"parameters":[82],"such":[84],"are":[87],"recovered":[88],"1-regularized":[90],"logistic":[91],"regression.":[92],"establish":[94],"bounds":[95],"difference":[98],"between":[99],"average":[101],"reward":[102],"estimated":[105],"original":[109],"policies":[110],"(regret)":[111],"terms":[113],"generalization":[116],"error":[117],"ergodic":[120],"coefficient":[121],"underlying":[124],"chain.":[126],"To":[127],"that":[128,148],"end,":[129],"we":[130],"combine":[131],"sample":[132,160],"complexity":[133],"theory":[134],"sensitivity":[136],"analysis":[137,146],"stationary":[140],"distribution":[141],"chains.":[144],"Our":[145],"suggests":[147],"achieve":[150],"regret":[151],"within":[152],"order":[153,164],"O(\u221a\u03b5),":[154],"it":[155],"suffices":[156],"use":[158],"training":[159],"size":[161],"\u03a9(log":[166],"n":[167,171],"\u00b7":[168],"poly(1/\u03b5)),":[169],"where":[170],"is":[172],"number":[174],"features.":[177],"demonstrate":[179],"effectiveness":[181],"our":[183],"method":[184],"synthetic":[187],"robot":[188],"navigation":[189],"example.":[190]},"counts_by_year":[{"year":2024,"cited_by_count":3},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":5},{"year":2019,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}