{"id":"https://openalex.org/W2080857937","doi":"https://doi.org/10.1109/tac.2015.2394951","title":"Optimization of Average Rewards of Time Nonhomogeneous Markov Chains","display_name":"Optimization of Average Rewards of Time Nonhomogeneous Markov Chains","publication_year":2015,"publication_date":"2015-01-21","ids":{"openalex":"https://openalex.org/W2080857937","doi":"https://doi.org/10.1109/tac.2015.2394951","mag":"2080857937"},"language":"en","primary_location":{"id":"doi:10.1109/tac.2015.2394951","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tac.2015.2394951","pdf_url":null,"source":{"id":"https://openalex.org/S184954342","display_name":"IEEE Transactions on Automatic Control","issn_l":"0018-9286","issn":["0018-9286","1558-2523","2334-3303"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automatic Control","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100635237","display_name":"Xi\u2010Ren Cao","orcid":"https://orcid.org/0000-0001-5165-8804"},"institutions":[{"id":"https://openalex.org/I183067930","display_name":"Shanghai Jiao Tong University","ror":"https://ror.org/0220qvk04","country_code":"CN","type":"education","lineage":["https://openalex.org/I183067930"]},{"id":"https://openalex.org/I889458895","display_name":"University of Hong Kong","ror":"https://ror.org/02zhqgq86","country_code":"HK","type":"education","lineage":["https://openalex.org/I889458895"]},{"id":"https://openalex.org/I200769079","display_name":"Hong Kong University of Science and Technology","ror":"https://ror.org/00q4vv597","country_code":"HK","type":"education","lineage":["https://openalex.org/I200769079"]}],"countries":["CN","HK"],"is_corresponding":true,"raw_author_name":"Xi-Ren Cao","raw_affiliation_strings":["Department of Finance and the Key Laboratory of System Control and Information Processing, Ministry of Education, Department of Automation, Hong Kong University of Science and Technology, Kowloon, Hongkong, China","Department Of Finance, Shanghai Jiao Tong University, Shanghai, China"],"affiliations":[{"raw_affiliation_string":"Department of Finance and the Key Laboratory of System Control and Information Processing, Ministry of Education, Department of Automation, Hong Kong University of Science and Technology, Kowloon, Hongkong, China","institution_ids":["https://openalex.org/I200769079","https://openalex.org/I889458895"]},{"raw_affiliation_string":"Department Of Finance, Shanghai Jiao Tong University, Shanghai, China","institution_ids":["https://openalex.org/I183067930"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5100635237"],"corresponding_institution_ids":["https://openalex.org/I183067930","https://openalex.org/I200769079","https://openalex.org/I889458895"],"apc_list":null,"apc_paid":null,"fwci":4.3144,"has_fulltext":false,"cited_by_count":26,"citation_normalized_percentile":{"value":0.94691294,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":89,"max":99},"biblio":{"volume":"60","issue":"7","first_page":"1841","last_page":"1856"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10142","display_name":"Formal Methods in Verification","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10768","display_name":"Electric Vehicles and Infrastructure","score":0.9810000061988831,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.8412808775901794},{"id":"https://openalex.org/keywords/ergodicity","display_name":"Ergodicity","score":0.7755753993988037},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.551217257976532},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.5305101275444031},{"id":"https://openalex.org/keywords/discrete-time-and-continuous-time","display_name":"Discrete time and continuous time","score":0.5009028911590576},{"id":"https://openalex.org/keywords/variable-order-markov-model","display_name":"Variable-order Markov model","score":0.49903035163879395},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.4945695102214813},{"id":"https://openalex.org/keywords/markov-property","display_name":"Markov property","score":0.47385719418525696},{"id":"https://openalex.org/keywords/transient","display_name":"Transient (computer programming)","score":0.44863057136535645},{"id":"https://openalex.org/keywords/markov-chain-mixing-time","display_name":"Markov chain mixing time","score":0.44658592343330383},{"id":"https://openalex.org/keywords/dynamic-programming","display_name":"Dynamic programming","score":0.43919575214385986},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.434776246547699},{"id":"https://openalex.org/keywords/markov-model","display_name":"Markov model","score":0.37209758162498474},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.1338791847229004}],"concepts":[{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.8412808775901794},{"id":"https://openalex.org/C201779956","wikidata":"https://www.wikidata.org/wiki/Q5426803","display_name":"Ergodicity","level":2,"score":0.7755753993988037},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.551217257976532},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5305101275444031},{"id":"https://openalex.org/C55689738","wikidata":"https://www.wikidata.org/wiki/Q15963867","display_name":"Discrete time and continuous time","level":2,"score":0.5009028911590576},{"id":"https://openalex.org/C54907487","wikidata":"https://www.wikidata.org/wiki/Q7915688","display_name":"Variable-order Markov model","level":4,"score":0.49903035163879395},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4945695102214813},{"id":"https://openalex.org/C189973286","wikidata":"https://www.wikidata.org/wiki/Q176695","display_name":"Markov property","level":4,"score":0.47385719418525696},{"id":"https://openalex.org/C2780799671","wikidata":"https://www.wikidata.org/wiki/Q17087362","display_name":"Transient (computer programming)","level":2,"score":0.44863057136535645},{"id":"https://openalex.org/C97074811","wikidata":"https://www.wikidata.org/wiki/Q6771322","display_name":"Markov chain mixing time","level":5,"score":0.44658592343330383},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.43919575214385986},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.434776246547699},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.37209758162498474},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.1338791847229004},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/tac.2015.2394951","is_oa":false,"landing_page_url":"https://doi.org/10.1109/tac.2015.2394951","pdf_url":null,"source":{"id":"https://openalex.org/S184954342","display_name":"IEEE Transactions on Automatic Control","issn_l":"0018-9286","issn":["0018-9286","1558-2523","2334-3303"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Automatic Control","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.41999998688697815,"display_name":"Reduced inequalities","id":"https://metadata.un.org/sdg/10"}],"awards":[{"id":"https://openalex.org/G4956378139","display_name":null,"funder_award_id":"61221003","funder_id":"https://openalex.org/F4320321001","funder_display_name":"National Natural Science Foundation of China"}],"funders":[{"id":"https://openalex.org/F4320321001","display_name":"National Natural Science Foundation of China","ror":"https://ror.org/01h0zpd94"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":23,"referenced_works":["https://openalex.org/W1494899432","https://openalex.org/W1506327958","https://openalex.org/W1596241917","https://openalex.org/W1967486764","https://openalex.org/W1977277767","https://openalex.org/W2010654234","https://openalex.org/W2024900792","https://openalex.org/W2028054171","https://openalex.org/W2084378215","https://openalex.org/W2102162493","https://openalex.org/W2116387273","https://openalex.org/W2119567691","https://openalex.org/W2119682789","https://openalex.org/W2143921394","https://openalex.org/W2152650468","https://openalex.org/W2163645125","https://openalex.org/W2171033527","https://openalex.org/W2315152390","https://openalex.org/W2334782222","https://openalex.org/W2484957131","https://openalex.org/W2611866857","https://openalex.org/W4253201987","https://openalex.org/W4254814937"],"related_works":["https://openalex.org/W2091522449","https://openalex.org/W2141396792","https://openalex.org/W2393764248","https://openalex.org/W2036451598","https://openalex.org/W4244907444","https://openalex.org/W2159877461","https://openalex.org/W3166133680","https://openalex.org/W4250947215","https://openalex.org/W607738810","https://openalex.org/W2171277320"],"abstract_inverted_index":{"We":[0,101,146,170],"study":[1,171],"the":[2,15,63,80,89,110,123,148,154,160,172,178,185],"optimization":[3,113,157],"of":[4,7,79,114,117,130],"average":[5,52,115,155],"rewards":[6,116],"discrete":[8],"time":[9],"nonhomogeneous":[10],"Markov":[11,38,118,132],"chains,":[12,39],"in":[13,65,91],"which":[14],"state":[16],"spaces,":[17],"transition":[18],"probabilities,":[19],"and":[20,45,69,76,158,162,168],"reward":[21,53,156,187],"functions":[22],"depend":[23,61],"on":[24,62,195],"time.":[25],"The":[26,51],"analysis":[27],"encounters":[28],"a":[29,104,131],"few":[30],"major":[31],"difficulties:":[32],"1)":[33,100],"Notions":[34],"crucial":[35],"to":[36,122,153,188],"homogeneous":[37],"such":[40],"as":[41],"ergodicity,":[42],"stationarity,":[43],"periodicity,":[44],"connectivity,":[46],"no":[47],"longer":[48],"apply;":[49],"2)":[50,145],"criterion":[54],"is":[55,73,109,198],"under-selective;":[56],"i.e,":[57],"it":[58],"does":[59],"not":[60,74,87],"decisions":[64],"any":[66,92,136],"finite":[67,93],"period,":[68],"thus":[70],"dynamic":[71],"programming":[72],"amenable;":[75],"3)":[77,169],"Because":[78],"under-selectivity,":[81],"an":[82],"optimal":[83,166],"average-reward":[84],"policy":[85],"may":[86],"be":[88,189],"best":[90],"period.":[94],"These":[95],"issues":[96],"are":[97],"resolved":[98],"by":[99],"discover":[102],"that":[103,125,183],"new":[105],"notion,":[106],"called":[107],"\u201cconfluencity\u201d,":[108],"base":[111],"for":[112,165,184],"chains.":[119],"Confluencity":[120],"refers":[121],"property":[124],"two":[126,137],"independent":[127],"sample":[128],"paths":[129],"chain":[133],"starting":[134],"from":[135],"different":[138],"initial":[139],"states":[140],"will":[141],"eventually":[142],"meet":[143],"together;":[144],"apply":[147],"direct-comparison":[149],"based":[150,194],"approach":[151],"[3]":[152],"obtain":[159],"necessary":[161],"sufficient":[163],"conditions":[164],"policies;":[167],"bias":[173,176,196],"optimality":[174],"with":[175],"measuring":[177],"transient":[179,186],"reward;":[180],"we":[181],"show":[182],"optimal,":[190],"one":[191],"additional":[192],"condition":[193],"potentials":[197],"required.":[199]},"counts_by_year":[{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":4},{"year":2019,"cited_by_count":6},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":5},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}