{"id":"https://openalex.org/W4399486984","doi":"https://doi.org/10.1109/tnnls.2024.3373749","title":"Off-Policy Prediction Learning: An Empirical Study of Online Algorithms","display_name":"Off-Policy Prediction Learning: An Empirical Study of Online Algorithms","publication_year":2024,"publication_date":"2024-06-10","ids":{"openalex":"https://openalex.org/W4399486984","doi":"https://doi.org/10.1109/tnnls.2024.3373749","pmid":"https://pubmed.ncbi.nlm.nih.gov/38857133"},"language":"en","primary_location":{"id":"doi:10.1109/tnnls.2024.3373749","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tnnls.2024.3373749","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://doi.org/10.1109/tnnls.2024.3373749","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5024789936","display_name":"Sina Ghiassian","orcid":"https://orcid.org/0000-0001-5853-5049"},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Sina Ghiassian","raw_affiliation_strings":["Department of Computing Science, University of Alberta, Edmonton, AB, Canada"],"raw_orcid":"https://orcid.org/0000-0001-5853-5049","affiliations":[{"raw_affiliation_string":"Department of Computing Science, University of Alberta, Edmonton, AB, Canada","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083856371","display_name":"Banafsheh Rafiee","orcid":"https://orcid.org/0000-0003-4641-7349"},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Banafsheh Rafiee","raw_affiliation_strings":["Reinforcement Learning and Artificial Intelligence (RLAI) Laboratory, University of Alberta, Edmonton, AB, Canada"],"raw_orcid":"https://orcid.org/0000-0003-4641-7349","affiliations":[{"raw_affiliation_string":"Reinforcement Learning and Artificial Intelligence (RLAI) Laboratory, University of Alberta, Edmonton, AB, Canada","institution_ids":["https://openalex.org/I154425047"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5004923102","display_name":"Richard S. Sutton","orcid":"https://orcid.org/0000-0002-3679-3415"},"institutions":[{"id":"https://openalex.org/I154425047","display_name":"University of Alberta","ror":"https://ror.org/0160cpw27","country_code":"CA","type":"education","lineage":["https://openalex.org/I154425047"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Richard S. Sutton","raw_affiliation_strings":["Department of Computing Science, University of Alberta, Edmonton, AB, Canada"],"raw_orcid":"https://orcid.org/0000-0002-3679-3415","affiliations":[{"raw_affiliation_string":"Department of Computing Science, University of Alberta, Edmonton, AB, Canada","institution_ids":["https://openalex.org/I154425047"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I154425047"],"apc_list":null,"apc_paid":null,"fwci":0.8469,"has_fulltext":false,"cited_by_count":2,"citation_normalized_percentile":{"value":0.74847601,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":95,"max":96},"biblio":{"volume":"36","issue":"3","first_page":"4477","last_page":"4491"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9987999796867371,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.590110182762146},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.5171448588371277},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.45956212282180786},{"id":"https://openalex.org/keywords/empirical-research","display_name":"Empirical research","score":0.4365385174751282},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4227994978427887},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1628870666027069},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.10490664839744568}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.590110182762146},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5171448588371277},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45956212282180786},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.4365385174751282},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4227994978427887},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1628870666027069},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.10490664839744568}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/tnnls.2024.3373749","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tnnls.2024.3373749","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},{"id":"pmid:38857133","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/38857133","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE transactions on neural networks and learning systems","raw_type":null}],"best_oa_location":{"id":"doi:10.1109/tnnls.2024.3373749","is_oa":true,"landing_page_url":"https://doi.org/10.1109/tnnls.2024.3373749","pdf_url":null,"source":{"id":"https://openalex.org/S4210175523","display_name":"IEEE Transactions on Neural Networks and Learning Systems","issn_l":"2162-237X","issn":["2162-237X","2162-2388"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Transactions on Neural Networks and Learning Systems","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320309949","display_name":"Canadian Institute for Advanced Research","ror":"https://ror.org/01sdtdd95"},{"id":"https://openalex.org/F4320314212","display_name":"Alberta Machine Intelligence Institute","ror":null},{"id":"https://openalex.org/F4320317153","display_name":"DeepMind","ror":"https://ror.org/00971b260"},{"id":"https://openalex.org/F4320334593","display_name":"Natural Sciences and Engineering Research Council of Canada","ror":"https://ror.org/01h531d29"}],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":73,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1487404723","https://openalex.org/W1522301498","https://openalex.org/W1547925194","https://openalex.org/W1646707810","https://openalex.org/W1835716857","https://openalex.org/W2026641456","https://openalex.org/W2075268401","https://openalex.org/W2109910161","https://openalex.org/W2121506959","https://openalex.org/W2121703796","https://openalex.org/W2145339207","https://openalex.org/W2146502635","https://openalex.org/W2150459019","https://openalex.org/W2198988526","https://openalex.org/W2473364827","https://openalex.org/W2606786028","https://openalex.org/W2746553466","https://openalex.org/W2761873684","https://openalex.org/W2785523195","https://openalex.org/W2898016531","https://openalex.org/W2898155524","https://openalex.org/W2919205453","https://openalex.org/W2919774326","https://openalex.org/W2948937890","https://openalex.org/W2950872548","https://openalex.org/W2953189830","https://openalex.org/W3011120880","https://openalex.org/W3041202696","https://openalex.org/W3080747787","https://openalex.org/W3181012797","https://openalex.org/W4200635299","https://openalex.org/W4214717370","https://openalex.org/W4281264198","https://openalex.org/W4297810816","https://openalex.org/W4297824605","https://openalex.org/W6609360466","https://openalex.org/W6612245137","https://openalex.org/W6629901736","https://openalex.org/W6630907848","https://openalex.org/W6631190155","https://openalex.org/W6632901617","https://openalex.org/W6635798015","https://openalex.org/W6638822454","https://openalex.org/W6677370284","https://openalex.org/W6677916933","https://openalex.org/W6677984395","https://openalex.org/W6679365036","https://openalex.org/W6681435938","https://openalex.org/W6682142764","https://openalex.org/W6683084483","https://openalex.org/W6697391007","https://openalex.org/W6715102896","https://openalex.org/W6725941776","https://openalex.org/W6729556111","https://openalex.org/W6732060498","https://openalex.org/W6734138181","https://openalex.org/W6736271032","https://openalex.org/W6737610669","https://openalex.org/W6738056049","https://openalex.org/W6747620207","https://openalex.org/W6748638692","https://openalex.org/W6752164047","https://openalex.org/W6755226438","https://openalex.org/W6755904448","https://openalex.org/W6758074843","https://openalex.org/W6759787782","https://openalex.org/W6762144173","https://openalex.org/W6762719077","https://openalex.org/W6763056091","https://openalex.org/W6775686901","https://openalex.org/W6779390608","https://openalex.org/W6805914596"],"related_works":["https://openalex.org/W2961085424","https://openalex.org/W4306674287","https://openalex.org/W4387369504","https://openalex.org/W3046775127","https://openalex.org/W4394896187","https://openalex.org/W3170094116","https://openalex.org/W4386462264","https://openalex.org/W3107602296","https://openalex.org/W4364306694","https://openalex.org/W4312192474"],"abstract_inverted_index":{"Off-policy":[0,247],"prediction-learning":[1],"the":[2,17,55,73,76,80,94,126,129,133,181,184,198,203,206,232,235,268,274,281,285,301,312,356,371,375,380,400,416],"value":[3],"function":[4,120],"for":[5,323],"one":[6,15],"policy":[7],"from":[8,79],"data":[9],"generated":[10],"while":[11],"following":[12],"another":[13],"policy-is":[14],"of":[16,149,186,205,234,259],"most":[18],"challenging":[19],"problems":[20],"in":[21,93,176,217,230,316,351],"reinforcement":[22],"learning.":[23,221],"This":[24],"article":[25],"makes":[26],"two":[27,60,253,361],"main":[28],"contributions:":[29],"1)":[30],"it":[31,58,98,157,389],"empirically":[32,111],"studies":[33],"11":[34,113],"off-policy":[35,86,114,144],"prediction":[36,87,115,269],"learning":[37,46,88,116,174],"algorithms":[38,89,102,117,244,345,402],"with":[39,75,118,160,374,408],"emphasis":[40],"on":[41,54,122,355],"their":[42],"sensitivity":[43],"to":[44,147,154,267,338,369,394],"parameters,":[45],"speed,":[47],"and":[48,51,67,132,165,257,262,294,387],"asymptotic":[49,341],"error":[50,78,342],"2)":[52],"based":[53],"empirical":[56,357],"results,":[57,358],"proposes":[59],"step-size":[61,362,381,411],"adaptation":[62,363,412],"methods":[63],"called":[64],"Step-size":[65,69],"Ratchet":[66,70,372,401],"Soft":[68],"that":[71,148,173,231,264,273,319,333,399],"help":[72],"algorithm":[74],"lowest":[77],"experimental":[81],"study":[82],"learn":[83,103,348],"faster.":[84],"Many":[85],"have":[90,339],"been":[91],"proposed":[92],"past":[95],"decade,":[96],"but":[97,309,346],"remains":[99],"unclear":[100],"which":[101,365],"faster":[104],"than":[105,343],"others.":[106],"In":[107,180],"this":[108],"article,":[109],"we":[110,359,366],"compare":[112],"linear":[119],"approximation":[121],"three":[123],"small":[124,143],"tasks:":[125],"Collision":[127,139],"task,":[128,131,183],"Rooms":[130,136,164,168,182,225],"High":[134,166,223],"Variance":[135,167,224],"task.":[137],"The":[138,163,222,243],"task":[140,226],"is":[141,178,227,277,320,328],"a":[142,317],"problem":[145],"analogous":[146],"an":[150,161],"autonomous":[151],"car":[152],"trying":[153],"predict":[155],"whether":[156],"will":[158],"collide":[159],"obstacle.":[162],"tasks":[169,324],"are":[170,246,265,297,403],"designed":[171],"such":[172,414],"fast":[175],"them":[177,407],"challenging.":[179],"product":[185,204,233],"importance":[187,207,286],"sampling":[188,208,287],"ratios":[189,236],"can":[190,237],"be":[191,213],"as":[192,194,239,241,304,306,370,383,385,415],"large":[193,240,384],".":[195,242],"To":[196],"control":[197],"high":[199,302,326],"variance":[200,282,303,327],"caused":[201],"by":[202,280,284,300,405],"ratios,":[209],"step":[210],"size":[211],"should":[212],"set":[214],"small,":[215],"which,":[216],"turn,":[218],"slows":[219],"down":[220,390],"more":[228,349],"extreme":[229],"become":[238],"considered":[245],"TD(":[248,335],"),":[249,291,293],"five":[250],"Gradient-TD":[251],"algorithms,":[252,255,308,364,373,413],"Emphatic-TD":[254],"Vtrace,":[256],"variants":[258],"Tree":[260,289],"Backup":[261],"ABQ":[263],"applicable":[266],"setting.":[270],"We":[271,331,397],"found":[272],"algorithms'":[275],"performance":[276],"highly":[278],"affected":[279,299],"induced":[283],"ratios.":[288],"Backup(":[290],"Vtrace(":[292],"ABTD(":[295],")":[296,336],"not":[298,329],"much":[305],"other":[307,344,409],"they":[310],"restrict":[311],"effective":[313,404],"bootstrapping":[314],"parameter":[315,382],"way":[318],"too":[321],"limiting":[322],"where":[325],"present.":[330],"observed":[332],"Emphatic":[334],"tends":[337],"lower":[340],"might":[347],"slowly":[350],"some":[352],"cases.":[353],"Based":[354],"propose":[360],"collectively":[367],"refer":[368],"same":[376],"underlying":[377],"idea:":[378],"keep":[379],"possible":[386],"ratchet":[388],"only":[391],"when":[392],"necessary":[393],"avoid":[395],"overshoot.":[396],"show":[398],"comparing":[406],"popular":[410],"Adam":[417],"optimizer.":[418]},"counts_by_year":[{"year":2025,"cited_by_count":2}],"updated_date":"2026-06-26T08:34:08.712188","created_date":"2025-10-10T00:00:00"}