{"id":"https://openalex.org/W2115211925","doi":null,"title":"Double Q-learning","display_name":"Double Q-learning","publication_year":2010,"publication_date":"2010-12-06","ids":{"openalex":"https://openalex.org/W2115211925","mag":"2115211925"},"language":"en","primary_location":{"id":"pmh:oai:cwi.nl:16889","doi":null,"is_oa":true,"landing_page_url":"https://ir.cwi.nl/pub/16889","pdf_url":"https://ir.cwi.nl/pub/16889/16889B.pdf","source":{"id":"https://openalex.org/S7407055335","display_name":"Centrum Wiskunde & Informatica (CWI), the national research institute for mathematics and computer science in the Netherlands","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://ir.cwi.nl/pub/16889/16889B.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5033135596","display_name":"Hado P. van Hasselt","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1341640284","display_name":"Centrum Wiskunde & Informatica","ror":"https://ror.org/00x7ekv49","country_code":"NL","type":"facility","lineage":["https://openalex.org/I1341640284","https://openalex.org/I2800991832","https://openalex.org/I4405262988"]}],"countries":["NL"],"is_corresponding":true,"raw_author_name":"Hado van Hasselt","raw_affiliation_strings":["Multi-agent and Adaptive Computation Group, Centrum Wiskunde & Informatica#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Multi-agent and Adaptive Computation Group, Centrum Wiskunde & Informatica#TAB#","institution_ids":["https://openalex.org/I1341640284"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5033135596"],"corresponding_institution_ids":["https://openalex.org/I1341640284"],"apc_list":null,"apc_paid":null,"fwci":2.5604,"has_fulltext":true,"cited_by_count":839,"citation_normalized_percentile":{"value":0.90986026,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":95,"max":100},"biblio":{"volume":"23","issue":null,"first_page":"2613","last_page":"2621"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/double-q-learning","display_name":"double Q-learning","score":0.9330000281333923},{"id":"https://openalex.org/keywords/q-learning","display_name":"Q-learning","score":0.9950000047683716},{"id":"https://openalex.org/keywords/off-policy-learning","display_name":"off-policy learning","score":0.8560000061988831},{"id":"https://openalex.org/keywords/stochastic-environments","display_name":"stochastic environments","score":0.5910000205039978}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8094415068626404},{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.688976526260376},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.6597425937652588},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.5763705968856812},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.5491889715194702},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5359877347946167},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4967196583747864},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4829068183898926},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4604509472846985},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.45862147212028503},{"id":"https://openalex.org/C141042865","wikidata":"https://www.wikidata.org/wiki/Q200125","display_name":"Expected value","level":2,"score":0.45361804962158203},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.44118884205818176},{"id":"https://openalex.org/C122123141","wikidata":"https://www.wikidata.org/wiki/Q176623","display_name":"Random variable","level":2,"score":0.41173070669174194},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3954257369041443},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3818305432796478},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.26938003301620483},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.24625161290168762},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:cwi.nl:16889","doi":null,"is_oa":true,"landing_page_url":"https://ir.cwi.nl/pub/16889","pdf_url":"https://ir.cwi.nl/pub/16889/16889B.pdf","source":{"id":"https://openalex.org/S7407055335","display_name":"Centrum Wiskunde & Informatica (CWI), the national research institute for mathematics and computer science in the Netherlands","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:cwi:oai:cwi.nl:16889","doi":null,"is_oa":true,"landing_page_url":"https://ir.cwi.nl/pub/16889","pdf_url":null,"source":{"id":"https://openalex.org/S4306401843","display_name":"Data Archiving and Networked Services (DANS)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I1322597698","host_organization_name":"Royal Netherlands Academy of Arts and Sciences","host_organization_lineage":["https://openalex.org/I1322597698"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferencepaper"},{"id":"mag:2115211925","doi":null,"is_oa":false,"landing_page_url":"https://ir.cwi.nl/pub/16889/16889B.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":"Neural Information Processing Systems","raw_type":null}],"best_oa_location":{"id":"pmh:oai:cwi.nl:16889","doi":null,"is_oa":true,"landing_page_url":"https://ir.cwi.nl/pub/16889","pdf_url":"https://ir.cwi.nl/pub/16889/16889B.pdf","source":{"id":"https://openalex.org/S7407055335","display_name":"Centrum Wiskunde & Informatica (CWI), the national research institute for mathematics and computer science in the Netherlands","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"info:eu-repo/semantics/conferenceObject"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320321800","display_name":"Nederlandse Organisatie voor Wetenschappelijk Onderzoek","ror":"https://ror.org/04jsz6e67"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W2115211925.pdf","grobid_xml":"https://content.openalex.org/works/W2115211925.grobid-xml"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W1515851193","https://openalex.org/W1626977535","https://openalex.org/W2015527784","https://openalex.org/W2027591961","https://openalex.org/W2043806097","https://openalex.org/W2048305092","https://openalex.org/W2075832535","https://openalex.org/W2103285838","https://openalex.org/W2107726111","https://openalex.org/W2117341272","https://openalex.org/W2118686230","https://openalex.org/W2120346334","https://openalex.org/W2121863487","https://openalex.org/W2122701159","https://openalex.org/W2129670787","https://openalex.org/W2138309709","https://openalex.org/W2147750403","https://openalex.org/W2150147323","https://openalex.org/W2150339816","https://openalex.org/W2158807713","https://openalex.org/W2159309155","https://openalex.org/W2165131254","https://openalex.org/W2341171179","https://openalex.org/W3011120880"],"related_works":["https://openalex.org/W7097180504","https://openalex.org/W2740912559","https://openalex.org/W2895095767","https://openalex.org/W3213562542","https://openalex.org/W2746553466","https://openalex.org/W4327673248","https://openalex.org/W3090993757","https://openalex.org/W4316039835","https://openalex.org/W3090826728","https://openalex.org/W3135838158"],"abstract_inverted_index":{"In":[0],"some":[1,116],"stochastic":[2],"environments":[3],"the":[4,36,44,56,79,85,102,107],"well-known":[5],"reinforcement":[6,97],"learning":[7,98],"algorithm":[8,104],"Q-learning":[9,34,89,120],"performs":[10,113,121],"very":[11],"poorly.":[12],"This":[13],"poor":[14],"performance":[15],"is":[16,31,71],"caused":[17],"by":[18],"large":[19],"overestimations":[20],"of":[21,63],"action":[22,38,47],"values,":[23],"which":[24,119],"result":[25],"from":[26],"a":[27,94],"positive":[28],"bias":[29],"that":[30,111],"introduced":[32],"because":[33],"uses":[35],"maximum":[37,45,57,80],"value":[39,59],"as":[40],"an":[41,51],"approximation":[42],"for":[43,60],"expected":[46,58,81],"value.":[48,82],"We":[49,83,100],"introduce":[50],"alternative":[52],"way":[53],"to":[54,73,88,90,106,124],"approximate":[55],"any":[61],"set":[62],"random":[64],"variables.":[65],"The":[66],"obtained":[67],"double":[68,86],"estimator":[69,87],"method":[70],"shown":[72],"sometimes":[74],"underestimate":[75],"rather":[76],"than":[77],"overestimate":[78],"apply":[84],"construct":[91],"Double":[92],"Q-learning,":[93],"new":[95,103],"off-policy":[96],"algorithm.":[99],"show":[101],"converges":[105],"optimal":[108],"policy":[109],"and":[110],"it":[112],"well":[114],"in":[115,118],"settings":[117],"poorly":[122],"due":[123],"its":[125],"overestimation.":[126]},"counts_by_year":[{"year":2025,"cited_by_count":42},{"year":2024,"cited_by_count":113},{"year":2023,"cited_by_count":111},{"year":2022,"cited_by_count":114},{"year":2021,"cited_by_count":157},{"year":2020,"cited_by_count":138},{"year":2019,"cited_by_count":84},{"year":2018,"cited_by_count":37},{"year":2017,"cited_by_count":17},{"year":2016,"cited_by_count":11},{"year":2015,"cited_by_count":3},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":4}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}