{"id":"https://openalex.org/W7134854134","doi":"https://doi.org/10.48550/arxiv.2603.07833","title":"Gradient Iterated Temporal-Difference Learning","display_name":"Gradient Iterated Temporal-Difference Learning","publication_year":2026,"publication_date":"2026-03-08","ids":{"openalex":"https://openalex.org/W7134854134","doi":"https://doi.org/10.48550/arxiv.2603.07833"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2603.07833","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5102678189","display_name":"Th\u00e9o Vincent","orcid":null},"institutions":[],"countries":[],"is_corresponding":true,"raw_author_name":"Vincent, Th\u00e9o","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5070007476","display_name":"KEVIN GERHARDT","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gerhardt, Kevin","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128632798","display_name":"Yogesh Tripathi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tripathi, Yogesh","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5114619957","display_name":"Habib Maraqten","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Maraqten, Habib","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128646269","display_name":"Adam White","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"White, Adam","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128647745","display_name":"Martha White","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"White, Martha","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5128650024","display_name":"Jan Peters","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peters, Jan","raw_affiliation_strings":[],"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5128633758","display_name":"Carlo D'Eramo","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"D'Eramo, Carlo","raw_affiliation_strings":[],"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":8,"corresponding_author_ids":["https://openalex.org/A5102678189"],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.4602999985218048,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.4602999985218048,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.4388999938964844,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.009499999694526196,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/iterated-function","display_name":"Iterated function","score":0.7760000228881836},{"id":"https://openalex.org/keywords/counterexample","display_name":"Counterexample","score":0.6626999974250793},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5475000143051147},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.4912000000476837},{"id":"https://openalex.org/keywords/gradient-method","display_name":"Gradient method","score":0.44209998846054077},{"id":"https://openalex.org/keywords/operator","display_name":"Operator (biology)","score":0.4325999915599823},{"id":"https://openalex.org/keywords/iterated-function-system","display_name":"Iterated function system","score":0.33469998836517334}],"concepts":[{"id":"https://openalex.org/C140479938","wikidata":"https://www.wikidata.org/wiki/Q5254619","display_name":"Iterated function","level":2,"score":0.7760000228881836},{"id":"https://openalex.org/C162838799","wikidata":"https://www.wikidata.org/wiki/Q596077","display_name":"Counterexample","level":2,"score":0.6626999974250793},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5475000143051147},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4968000054359436},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.4912000000476837},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4456999897956848},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.44209998846054077},{"id":"https://openalex.org/C17020691","wikidata":"https://www.wikidata.org/wiki/Q139677","display_name":"Operator (biology)","level":5,"score":0.4325999915599823},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42829999327659607},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4163999855518341},{"id":"https://openalex.org/C168460219","wikidata":"https://www.wikidata.org/wiki/Q1502167","display_name":"Iterated function system","level":3,"score":0.33469998836517334},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.31630000472068787},{"id":"https://openalex.org/C120822770","wikidata":"https://www.wikidata.org/wiki/Q5156355","display_name":"Competitive learning","level":3,"score":0.30640000104904175},{"id":"https://openalex.org/C40506919","wikidata":"https://www.wikidata.org/wiki/Q7452469","display_name":"Sequence learning","level":2,"score":0.2976999878883362},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.296099990606308},{"id":"https://openalex.org/C24138899","wikidata":"https://www.wikidata.org/wiki/Q17141258","display_name":"Instance-based learning","level":3,"score":0.2903999984264374},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.28439998626708984},{"id":"https://openalex.org/C115903097","wikidata":"https://www.wikidata.org/wiki/Q7094097","display_name":"Online machine learning","level":3,"score":0.260699987411499},{"id":"https://openalex.org/C91873725","wikidata":"https://www.wikidata.org/wiki/Q3445816","display_name":"Function approximation","level":3,"score":0.2547000050544739}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2603.07833","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2603.07833","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2603.07833","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"article"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2603.07833","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Temporal-difference":[0],"(TD)":[1],"learning":[2,26,72,81,87,149,185],"is":[3,44,106],"highly":[4],"effective":[5],"at":[6],"controlling":[7],"and":[8],"evaluating":[9],"an":[10],"agent's":[11],"long-term":[12],"outcomes.":[13],"Most":[14],"approaches":[15],"in":[16,101,120],"this":[17,40,59,125,143,175],"paradigm":[18],"implement":[19],"a":[20,96,139,161,183,196],"semi-gradient":[21,76,133,169,188],"update":[22,43],"to":[23,46,57,69,75,84,108,131,159],"boost":[24],"the":[25,32,35,86,110,113,117,121,152],"speed,":[27],"which":[28],"consists":[29],"of":[30,34,42,89,98,112],"ignoring":[31],"gradient":[33,163,203],"bootstrapped":[36],"estimate.":[37],"While":[38,123],"popular,":[39],"type":[41],"prone":[45],"divergence,":[47],"as":[48,135],"Baird's":[49],"counterexample":[50],"illustrates.":[51],"Gradient":[52,178],"TD":[53,80,90,148,164,204],"methods":[54,189,205],"were":[55],"introduced":[56],"overcome":[58],"issue,":[60],"but":[61],"have":[62],"not":[63],"been":[64],"widely":[65],"used,":[66],"potentially":[67],"due":[68,130],"issues":[70],"with":[71,168],"speed":[73,88,186],"compared":[74],"methods.":[77,91,170],"Recently,":[78],"iterated":[79,147],"was":[82],"developed":[83],"increase":[85],"For":[92],"that,":[93],"it":[94],"learns":[95],"sequence":[97],"action-value":[99],"functions":[100],"parallel,":[102],"where":[103],"each":[104,136],"function":[105,119,137],"optimized":[107],"represent":[109],"application":[111],"Bellman":[114],"operator":[115],"over":[116,154],"previous":[118],"sequence.":[122],"promising,":[124],"algorithm":[126],"can":[127],"be":[128],"unstable":[129],"its":[132],"nature,":[134],"tracks":[138],"moving":[140,156],"target.":[141],"In":[142],"work,":[144],"we":[145],"modify":[146],"by":[150],"computing":[151],"gradients":[153],"those":[155],"targets,":[157],"aiming":[158],"build":[160],"powerful":[162],"method":[165],"that":[166,174,198],"competes":[167],"Our":[171],"evaluation":[172],"reveals":[173],"algorithm,":[176],"called":[177],"Iterated":[179],"Temporal-Difference":[180],"learning,":[181],"has":[182,206],"competitive":[184],"against":[187],"across":[190],"various":[191],"benchmarks,":[192],"including":[193],"Atari":[194],"games,":[195],"result":[197],"no":[199],"prior":[200],"work":[201],"on":[202],"demonstrated.":[207]},"counts_by_year":[],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2026-03-11T00:00:00"}