{"id":"https://openalex.org/W3035536593","doi":"https://doi.org/10.26153/tsw/14853","title":"Reducing sampling error in batch temporal difference learning","display_name":"Reducing sampling error in batch temporal difference learning","publication_year":2020,"publication_date":"2020-05-01","ids":{"openalex":"https://openalex.org/W3035536593","doi":"https://doi.org/10.26153/tsw/14853","mag":"3035536593"},"language":"en","primary_location":{"id":"mag:3035536593","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2008.06738","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2008.06738","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5070355024","display_name":"Brahma S. Pavse","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I86519309","display_name":"The University of Texas at Austin","ror":"https://ror.org/00hj54h04","country_code":"US","type":"education","lineage":["https://openalex.org/I86519309"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Pavse, Brahma Suneil","raw_affiliation_strings":["University of Texas at Austin"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Texas at Austin","institution_ids":["https://openalex.org/I86519309"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5070355024"],"corresponding_institution_ids":["https://openalex.org/I86519309"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":5,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"1","issue":null,"first_page":"7543","last_page":"7552"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"temporal difference learning","score":0.9210000038146973},{"id":"https://openalex.org/keywords/offline-reinforcement-learning","display_name":"offline reinforcement learning","score":0.7730000019073486},{"id":"https://openalex.org/keywords/sampling-error","display_name":"sampling error","score":0.7910000085830688},{"id":"https://openalex.org/keywords/importance-sampling","display_name":"importance sampling","score":0.9100000262260437},{"id":"https://openalex.org/keywords/value-function-estimation","display_name":"value function estimation","score":0.9039999842643738},{"id":"https://openalex.org/keywords/fixed-point-convergence","display_name":"fixed point convergence","score":0.781000018119812},{"id":"https://openalex.org/keywords/mean-squared-error","display_name":"mean squared error","score":0.6740000247955322}],"concepts":[{"id":"https://openalex.org/C183115368","wikidata":"https://www.wikidata.org/wiki/Q856577","display_name":"Weighting","level":2,"score":0.6943718791007996},{"id":"https://openalex.org/C8642999","wikidata":"https://www.wikidata.org/wiki/Q4171168","display_name":"Hyperparameter","level":2,"score":0.6586651802062988},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.6570928692817688},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6061168909072876},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.5911877155303955},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5647308230400085},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.48066914081573486},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.466370165348053},{"id":"https://openalex.org/C98385598","wikidata":"https://www.wikidata.org/wiki/Q1339385","display_name":"Empirical distribution function","level":2,"score":0.46602746844291687},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.4556666910648346},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.3794468939304352},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.3539623022079468},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.29408252239227295},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C71924100","wikidata":"https://www.wikidata.org/wiki/Q11190","display_name":"Medicine","level":0,"score":0.0},{"id":"https://openalex.org/C126838900","wikidata":"https://www.wikidata.org/wiki/Q77604","display_name":"Radiology","level":1,"score":0.0},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C106131492","wikidata":"https://www.wikidata.org/wiki/Q3072260","display_name":"Filter (signal processing)","level":2,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"mag:3035536593","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2008.06738","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.26153/tsw/14853","doi":"https://doi.org/10.26153/tsw/14853","is_oa":true,"landing_page_url":"https://doi.org/10.26153/tsw/14853","pdf_url":null,"source":{"id":"https://openalex.org/S4306402198","display_name":"Texas Digital Library (University of Texas)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I86519309","host_organization_name":"The University of Texas at Austin","host_organization_lineage":["https://openalex.org/I86519309"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Dissertation"}],"best_oa_location":{"id":"mag:3035536593","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/2008.06738","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2949488369","https://openalex.org/W2995999665","https://openalex.org/W7099751774","https://openalex.org/W7097553111","https://openalex.org/W3146077118","https://openalex.org/W3201874992","https://openalex.org/W2401403926","https://openalex.org/W7098940975","https://openalex.org/W7100174464","https://openalex.org/W7133496047"],"abstract_inverted_index":{"Temporal":[0],"difference":[1],"(TD)":[2],"learning":[3,175],"is":[4,61],"one":[5],"of":[6,10,19,30,38,67,80,104,139,159,168,180],"the":[7,17,27,56,65,73,77,81,84,101,110,120,123,127,137,196],"main":[8],"foundations":[9],"modern":[11],"reinforcement":[12],"learning.":[13],"This":[14],"thesis":[15],"studies":[16],"use":[18],"TD(0),":[20],"a":[21,31,36,140,149,156,164,178],"canonical":[22],"TD":[23],"algorithm,":[24],"to":[25,50,64,117,148],"estimate":[26,142],"value":[28,53,173,187],"function":[29,54,174,188],"given":[32],"evaluation":[33,85,167],"policy":[34,93],"from":[35],"batch":[37,42,74,111,158,172],"data.":[39,160],"In":[40],"this":[41,89],"setting,":[43],"we":[44,91,162],"show":[45,183],"that":[46,69,145,184],"TD(0)":[47,154,198],"may":[48],"converge":[49],"an":[51,59],"inaccurate":[52],"because":[55],"update":[57],"following":[58,132],"action":[60,70,82],"weighted":[62],"according":[63],"number":[66],"times":[68],"occurred":[71],"in":[72,106,109,177],"--":[75],"not":[76],"true":[78],"probability":[79],"under":[83],"policy.":[86],"To":[87],"address":[88],"limitation,":[90],"introduce":[92],"sampling":[94,116],"error":[95,194],"corrected-TD(0)":[96],"(PSEC-TD(0)).":[97],"PSEC-TD(0)":[98,146,169,185],"first":[99],"estimates":[100,189],"empirical":[102,124,166],"distribution":[103],"actions":[105],"each":[107,133],"state":[108],"and":[112,126,143,182],"then":[113],"uses":[114],"importance":[115],"correct":[118,128],"for":[119,130,155],"mismatch":[121],"between":[122],"weighting":[125,129],"updates":[131],"action.":[134],"We":[135],"refine":[136],"concept":[138],"certainty-equivalence":[141],"argue":[144],"converges":[147],"more":[150],"desirable":[151],"fixed-point":[152],"than":[153,195],"fixed":[157],"Finally,":[161],"conduct":[163],"thorough":[165],"on":[170],"three":[171],"tasks":[176],"variety":[179],"settings":[181],"produces":[186],"with":[190],"lower":[191],"mean":[192],"squared":[193],"standard":[197],"algorithm":[199]},"counts_by_year":[{"year":2023,"cited_by_count":2},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":2}],"updated_date":"2026-11-03T10:50:19.383937","created_date":"2020-06-19T00:00:00"}