{"id":"https://openalex.org/W2964988441","doi":"https://doi.org/10.1137/20m1311971","title":"Finite-Time Performance of Distributed Temporal-Difference Learning with Linear Function Approximation","display_name":"Finite-Time Performance of Distributed Temporal-Difference Learning with Linear Function Approximation","publication_year":2021,"publication_date":"2021-01-01","ids":{"openalex":"https://openalex.org/W2964988441","doi":"https://doi.org/10.1137/20m1311971","mag":"2964988441"},"language":"en","primary_location":{"id":"doi:10.1137/20m1311971","is_oa":true,"landing_page_url":"https://doi.org/10.1137/20m1311971","pdf_url":"https://epubs.siam.org/doi/pdf/10.1137/20M1311971","source":{"id":"https://openalex.org/S4210229561","display_name":"SIAM Journal on Mathematics of Data Science","issn_l":"2577-0187","issn":["2577-0187"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Mathematics of Data Science","raw_type":"journal-article"},"type":"preprint","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://epubs.siam.org/doi/pdf/10.1137/20M1311971","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5035207859","display_name":"Thinh T. Doan","orcid":"https://orcid.org/0000-0001-5135-3429"},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Thinh T. Doan","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0001-5135-3429","affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021806638","display_name":"Siva Theja Maguluri","orcid":"https://orcid.org/0000-0002-5797-1639"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Siva Theja Maguluri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5041443633","display_name":"Justin Romberg","orcid":"https://orcid.org/0000-0002-6616-197X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Justin Romberg","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5035207859"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.8399,"has_fulltext":false,"cited_by_count":19,"citation_normalized_percentile":{"value":0.7756616,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":99},"biblio":{"volume":"3","issue":"1","first_page":"298","last_page":"320"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.984000027179718,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10524","display_name":"Traffic control and management","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7529146075248718},{"id":"https://openalex.org/keywords/lambda","display_name":"Lambda","score":0.6695952415466309},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.6577653884887695},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.6453644037246704},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.6104981303215027},{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"Temporal difference learning","score":0.6100306510925293},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.5659918189048767},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5645732879638672},{"id":"https://openalex.org/keywords/upper-and-lower-bounds","display_name":"Upper and lower bounds","score":0.533820629119873},{"id":"https://openalex.org/keywords/constant","display_name":"Constant (computer programming)","score":0.5090848803520203},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.5056958198547363},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.502825915813446},{"id":"https://openalex.org/keywords/q-learning","display_name":"Q-learning","score":0.5025639533996582},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.496919184923172},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.412632554769516},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.41163069009780884},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.35355156660079956},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.23351770639419556},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.21366816759109497},{"id":"https://openalex.org/keywords/mathematical-analysis","display_name":"Mathematical analysis","score":0.16037911176681519},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.1198943555355072},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.107462078332901},{"id":"https://openalex.org/keywords/physics","display_name":"Physics","score":0.09790229797363281}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7529146075248718},{"id":"https://openalex.org/C2778113609","wikidata":"https://www.wikidata.org/wiki/Q10897","display_name":"Lambda","level":2,"score":0.6695952415466309},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.6577653884887695},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.6453644037246704},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.6104981303215027},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.6100306510925293},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.5659918189048767},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5645732879638672},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.533820629119873},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.5090848803520203},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5056958198547363},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.502825915813446},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.5025639533996582},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.496919184923172},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.412632554769516},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.41163069009780884},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.35355156660079956},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.23351770639419556},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.21366816759109497},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.16037911176681519},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.1198943555355072},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.107462078332901},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.09790229797363281},{"id":"https://openalex.org/C120665830","wikidata":"https://www.wikidata.org/wiki/Q14620","display_name":"Optics","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1137/20m1311971","is_oa":true,"landing_page_url":"https://doi.org/10.1137/20m1311971","pdf_url":"https://epubs.siam.org/doi/pdf/10.1137/20M1311971","source":{"id":"https://openalex.org/S4210229561","display_name":"SIAM Journal on Mathematics of Data Science","issn_l":"2577-0187","issn":["2577-0187"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Mathematics of Data Science","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:1907.12530","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1907.12530","pdf_url":"https://arxiv.org/pdf/1907.12530","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:2964988441","is_oa":true,"landing_page_url":"https://arxiv.org/abs/1907.12530","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1907.12530","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1907.12530","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"doi:10.1137/20m1311971","is_oa":true,"landing_page_url":"https://doi.org/10.1137/20m1311971","pdf_url":"https://epubs.siam.org/doi/pdf/10.1137/20M1311971","source":{"id":"https://openalex.org/S4210229561","display_name":"SIAM Journal on Mathematics of Data Science","issn_l":"2577-0187","issn":["2577-0187"],"is_oa":true,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Mathematics of Data Science","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.4000000059604645,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2964988441.pdf","grobid_xml":"https://content.openalex.org/works/W2964988441.grobid-xml"},"referenced_works_count":33,"referenced_works":["https://openalex.org/W594357522","https://openalex.org/W1507222174","https://openalex.org/W1918371733","https://openalex.org/W1966221033","https://openalex.org/W2071983464","https://openalex.org/W2073384958","https://openalex.org/W2075268401","https://openalex.org/W2101130703","https://openalex.org/W2119112357","https://openalex.org/W2121863487","https://openalex.org/W2131600418","https://openalex.org/W2139418546","https://openalex.org/W2144672231","https://openalex.org/W2145339207","https://openalex.org/W2154761920","https://openalex.org/W2158091072","https://openalex.org/W2159826786","https://openalex.org/W2165418472","https://openalex.org/W2167485994","https://openalex.org/W2173945562","https://openalex.org/W2257979135","https://openalex.org/W2395162158","https://openalex.org/W2504148212","https://openalex.org/W2610857016","https://openalex.org/W2788115019","https://openalex.org/W2798888671","https://openalex.org/W2944956041","https://openalex.org/W2962771678","https://openalex.org/W2962856794","https://openalex.org/W2964084913","https://openalex.org/W2964123095","https://openalex.org/W2964258803","https://openalex.org/W2970400802"],"related_works":["https://openalex.org/W3136903997","https://openalex.org/W2966363432","https://openalex.org/W2964005211","https://openalex.org/W2121863487","https://openalex.org/W1918371733","https://openalex.org/W594357522","https://openalex.org/W2913326990","https://openalex.org/W2100677568","https://openalex.org/W2075268401","https://openalex.org/W2963747324","https://openalex.org/W2963695783","https://openalex.org/W2963649943","https://openalex.org/W2944956041","https://openalex.org/W1994616650","https://openalex.org/W3176355038","https://openalex.org/W3008618380","https://openalex.org/W3152894662","https://openalex.org/W166862392","https://openalex.org/W1534630452","https://openalex.org/W1521003796"],"abstract_inverted_index":{"We":[0,158],"study":[1],"the":[2,19,35,52,62,84,102,126,132,139,165,169,176,179,182,186,204,207,215,225,231,236,241,253,256],"policy":[3],"evaluation":[4],"problem":[5],"in":[6,22,120,141,203,224,255],"multi-agent":[7],"reinforcement":[8],"learning,":[9],"modeled":[10],"by":[11],"a":[12,23,27,47,80,99,173,194],"Markov":[13,134],"decision":[14],"process.":[15],"In":[16],"this":[17,105],"problem,":[18],"agents":[20,53],"operate":[21],"common":[24],"environment":[25],"under":[26],"fixed":[28],"control":[29],"policy,":[30],"working":[31],"together":[32],"to":[33,97,124,235,249],"discover":[34],"value":[36,63,242],"(global":[37],"discounted":[38],"accumulative":[39],"reward)":[40],"associated":[41],"with":[42,67],"each":[43,74],"environmental":[44],"state.":[45],"Over":[46],"series":[48],"of":[49,61,83,104,131,168,175,209,227,240],"time":[50,129,188],"steps,":[51],"act,":[54],"get":[55],"rewarded,":[56],"update":[57,72],"their":[58,68],"local":[59,71],"estimate":[60],"function,":[64,243],"then":[65],"communicate":[66],"neighbors.":[69],"The":[70,117],"at":[73,151,155],"agent":[75],"can":[76],"be":[77],"interpreted":[78],"as":[79,172],"distributed":[81,106],"variant":[82],"popular":[85],"temporal":[86],"difference":[87],"learning":[88],"methods":[89],"{\\sf":[90,107],"TD}$":[91],"(\u03bb)$.":[92],"Our":[93,190,211],"main":[94],"contribution":[95],"is":[96,123,144,148],"provide":[98,159,193],"finite-analysis":[100],"on":[101,164],"performance":[103],"TD}$(\u03bb)$":[108],"algorithm":[109,143,257],"for":[110,197],"both":[111],"constant":[112,183],"and":[113,185,220],"time-varying":[114],"step":[115],"sizes.":[116],"key":[118],"idea":[119],"our":[121,142],"analysis":[122],"use":[125],"geometric":[127],"mixing":[128,187],"$\u03c4$":[130],"underlying":[133],"chain,":[135],"that":[136,199],"is,":[137],"although":[138],"\"noise\"":[140],"Markovian,":[145],"its":[146],"dependence":[147],"very":[149],"weak":[150],"samples":[152],"spaced":[153],"out":[154],"every":[156],"$\u03c4$.":[157,189],"an":[160],"explicit":[161],"upper":[162,212],"bound":[163,213],"convergence":[166,221,251],"rate":[167],"proposed":[170],"method":[171],"function":[174],"network":[177],"topology,":[178],"discount":[180],"factor,":[181],"$\u03bb$,":[184],"results":[191],"also":[192],"mathematical":[195],"explanation":[196],"observations":[198],"have":[200],"appeared":[201],"previously":[202],"literature":[205],"about":[206],"choice":[208,226],"$\u03bb$.":[210,228],"illustrates":[214],"trade-off":[216],"between":[217],"approximation":[218,239],"accuracy":[219],"speed":[222],"implicit":[223],"When":[229],"$\u03bb=1$,":[230],"solution":[232],"will":[233],"correspond":[234],"best":[237],"possible":[238],"while":[244],"choosing":[245],"$\u03bb=":[246],"0$":[247],"leads":[248],"faster":[250],"when":[252],"noise":[254],"has":[258],"large":[259],"variance.":[260]},"counts_by_year":[{"year":2021,"cited_by_count":6},{"year":2020,"cited_by_count":9},{"year":2019,"cited_by_count":3},{"year":2018,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}