{"id":"https://openalex.org/W7133496047","doi":"https://doi.org/10.48550/arxiv.2603.02577","title":"Towards Parameter-Free Temporal Difference Learning","display_name":"Towards Parameter-Free Temporal Difference Learning","publication_year":2026,"publication_date":"2026-03-03","ids":{"openalex":"https://openalex.org/W7133496047","doi":"https://doi.org/10.48550/arxiv.2603.02577"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2603.02577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.02577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2603.02577","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5094275020","display_name":"Yunxiang Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yunxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101466411","display_name":"Mark Schmidt","orcid":"https://orcid.org/0000-0003-1129-5273"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Schmidt, Mark","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5090133726","display_name":"Reza Babanezhad","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Babanezhad, Reza","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128098465","display_name":"Sharan Vaswani","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Vaswani, Sharan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9064000248908997,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9064000248908997,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.009399999864399433,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.006200000178068876,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/independent-and-identically-distributed-random-variables","display_name":"Independent and identically distributed random variables","score":0.6226000189781189},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.5073000192642212},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.4966999888420105},{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"Temporal difference learning","score":0.48739999532699585},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.4869999885559082},{"id":"https://openalex.org/keywords/exponential-function","display_name":"Exponential function","score":0.46000000834465027},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.45500001311302185},{"id":"https://openalex.org/keywords/mixing","display_name":"Mixing (physics)","score":0.4465000033378601},{"id":"https://openalex.org/keywords/covariance","display_name":"Covariance","score":0.4244999885559082}],"concepts":[{"id":"https://openalex.org/C141513077","wikidata":"https://www.wikidata.org/wiki/Q378542","display_name":"Independent and identically distributed random variables","level":3,"score":0.6226000189781189},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5812000036239624},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.5073000192642212},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.505299985408783},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.4966999888420105},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.48739999532699585},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.4869999885559082},{"id":"https://openalex.org/C151376022","wikidata":"https://www.wikidata.org/wiki/Q168698","display_name":"Exponential function","level":2,"score":0.46000000834465027},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.45500001311302185},{"id":"https://openalex.org/C138777275","wikidata":"https://www.wikidata.org/wiki/Q6884054","display_name":"Mixing (physics)","level":2,"score":0.4465000033378601},{"id":"https://openalex.org/C178650346","wikidata":"https://www.wikidata.org/wiki/Q201984","display_name":"Covariance","level":2,"score":0.4244999885559082},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.41110000014305115},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4106000065803528},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4074999988079071},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3767000138759613},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3765999972820282},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.3707999885082245},{"id":"https://openalex.org/C52740198","wikidata":"https://www.wikidata.org/wiki/Q1539564","display_name":"Importance sampling","level":3,"score":0.36250001192092896},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3529999852180481},{"id":"https://openalex.org/C55974624","wikidata":"https://www.wikidata.org/wiki/Q1188504","display_name":"Exponential family","level":2,"score":0.33480000495910645},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.33180001378059387},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.32690000534057617},{"id":"https://openalex.org/C68387754","wikidata":"https://www.wikidata.org/wiki/Q7271585","display_name":"Schedule","level":2,"score":0.3125},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.3095000088214874},{"id":"https://openalex.org/C75235859","wikidata":"https://www.wikidata.org/wiki/Q582659","display_name":"Exponential growth","level":2,"score":0.2924000024795532},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.29109999537467957},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.2775000035762787},{"id":"https://openalex.org/C110405555","wikidata":"https://www.wikidata.org/wiki/Q1192209","display_name":"Stationary process","level":2,"score":0.25699999928474426},{"id":"https://openalex.org/C158693339","wikidata":"https://www.wikidata.org/wiki/Q190524","display_name":"Eigenvalues and eigenvectors","level":2,"score":0.25369998812675476},{"id":"https://openalex.org/C148043351","wikidata":"https://www.wikidata.org/wiki/Q4456944","display_name":"Current (fluid)","level":2,"score":0.25200000405311584}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2603.02577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.02577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2603.02577","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.02577","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Temporal":[0],"difference":[1],"(TD)":[2],"learning":[3],"is":[4],"a":[5,126,162,175],"fundamental":[6],"algorithm":[7,35,135,165,173],"for":[8,152],"estimating":[9],"value":[10],"functions":[11],"in":[12,45],"reinforcement":[13],"learning.":[14],"Recent":[15],"finite-time":[16],"analyses":[17,71],"of":[18,53,62,140,189],"TD":[19],"with":[20,95,166],"linear":[21],"function":[22],"approximation":[23],"quantify":[24],"its":[25],"theoretical":[26],"convergence":[27,177],"rate.":[28],"However,":[29],"they":[30],"often":[31],"require":[32,138],"setting":[33],"the":[34,50,54,59,63,79,96,102,116,120,130,133,148,153,157],"parameters":[36],"using":[37],"problem-dependent":[38,141],"quantities":[39,142],"that":[40],"are":[41],"difficult":[42],"to":[43,179],"estimate":[44],"practice":[46],"--":[47],"such":[48,143],"as":[49,144],"minimum":[51],"eigenvalue":[52],"feature":[55],"covariance":[56],"(\\(\u03c9\\))":[57],"or":[58,187,191],"mixing":[60],"time":[61],"underlying":[64],"Markov":[65],"chain":[66],"(\\(\u03c4_{\\text{mix}}\\)).":[67],"In":[68,129,156],"addition,":[69],"some":[70],"rely":[72],"on":[73],"nonstandard":[74],"and":[75,83,110,119,146],"impractical":[76],"modifications,":[77],"exacerbating":[78],"gap":[80],"between":[81],"theory":[82],"practice.":[84],"To":[85],"address":[86],"these":[87],"limitations,":[88],"we":[89,160],"use":[90],"an":[91,167],"exponential":[92,168],"step-size":[93,169],"schedule":[94],"standard":[97],"TD(0)":[98,164],"algorithm.":[99],"We":[100],"analyze":[101],"resulting":[103,172],"method":[104],"under":[105],"two":[106],"sampling":[107,114,124],"regimes:":[108],"independent":[109],"identically":[111],"distributed":[112],"(i.i.d.)":[113],"from":[115],"stationary":[117],"distribution,":[118],"more":[121],"practical":[122],"Markovian":[123,158],"along":[125],"single":[127],"trajectory.":[128],"i.i.d.\\":[131],"setting,":[132,159],"proposed":[134],"does":[136],"not":[137],"knowledge":[139,188],"\\(\u03c9\\),":[145],"attains":[147],"optimal":[149],"bias-variance":[150],"trade-off":[151],"last":[154],"iterate.":[155],"propose":[161],"regularized":[163],"schedule.":[170],"The":[171],"achieves":[174],"comparable":[176],"rate":[178],"prior":[180],"works,":[181],"without":[182],"requiring":[183],"projections,":[184],"iterate":[185],"averaging,":[186],"\\(\u03c4_{\\text{mix}}\\)":[190],"\\(\u03c9\\).":[192]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-03-05T00:00:00"}