{"id":"https://openalex.org/W2973184394","doi":"https://doi.org/10.1109/access.2019.2941229","title":"Q-Learning Algorithms: A Comprehensive Classification and Applications","display_name":"Q-Learning Algorithms: A Comprehensive Classification and Applications","publication_year":2019,"publication_date":"2019-01-01","ids":{"openalex":"https://openalex.org/W2973184394","doi":"https://doi.org/10.1109/access.2019.2941229","mag":"2973184394"},"language":"en","primary_location":{"id":"doi:10.1109/access.2019.2941229","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2941229","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08836506.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","doaj"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08836506.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067151609","display_name":"Beakcheol Jang","orcid":"https://orcid.org/0000-0002-3911-5935"},"institutions":[{"id":"https://openalex.org/I157264075","display_name":"Sangmyung University","ror":"https://ror.org/01x4whx42","country_code":"KR","type":"education","lineage":["https://openalex.org/I157264075"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Beakcheol Jang","raw_affiliation_strings":["Department of Computer Science, Sangmyung University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0002-3911-5935","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Sangmyung University, Seoul, South Korea","institution_ids":["https://openalex.org/I157264075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5050629182","display_name":"Myeonghwi Kim","orcid":null},"institutions":[{"id":"https://openalex.org/I157264075","display_name":"Sangmyung University","ror":"https://ror.org/01x4whx42","country_code":"KR","type":"education","lineage":["https://openalex.org/I157264075"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Myeonghwi Kim","raw_affiliation_strings":["Department of Computer Science, Sangmyung University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0001-8036-4434","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Sangmyung University, Seoul, South Korea","institution_ids":["https://openalex.org/I157264075"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5080143856","display_name":"Gaspard Harerimana","orcid":"https://orcid.org/0000-0003-4761-562X"},"institutions":[{"id":"https://openalex.org/I157264075","display_name":"Sangmyung University","ror":"https://ror.org/01x4whx42","country_code":"KR","type":"education","lineage":["https://openalex.org/I157264075"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Gaspard Harerimana","raw_affiliation_strings":["Department of Computer Science, Sangmyung University, Seoul, South Korea"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Computer Science, Sangmyung University, Seoul, South Korea","institution_ids":["https://openalex.org/I157264075"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100726106","display_name":"Jong Wook Kim","orcid":"https://orcid.org/0000-0001-8373-1893"},"institutions":[{"id":"https://openalex.org/I157264075","display_name":"Sangmyung University","ror":"https://ror.org/01x4whx42","country_code":"KR","type":"education","lineage":["https://openalex.org/I157264075"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Jong Wook Kim","raw_affiliation_strings":["Department of Computer Science, Sangmyung University, Seoul, South Korea"],"raw_orcid":"https://orcid.org/0000-0001-8373-1893","affiliations":[{"raw_affiliation_string":"Department of Computer Science, Sangmyung University, Seoul, South Korea","institution_ids":["https://openalex.org/I157264075"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I157264075"],"apc_list":{"value":1850,"currency":"USD","value_usd":1850},"apc_paid":{"value":1850,"currency":"USD","value_usd":1850},"fwci":22.1806,"has_fulltext":true,"cited_by_count":624,"citation_normalized_percentile":{"value":0.99536423,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":90,"max":100},"biblio":{"volume":"7","issue":null,"first_page":"133653","last_page":"133667"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9980999827384949,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.9829999804496765,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12761","display_name":"Data Stream Mining Techniques","score":0.9829000234603882,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.798820972442627},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7622866630554199},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.7387844920158386},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7370942234992981},{"id":"https://openalex.org/keywords/instance-based-learning","display_name":"Instance-based learning","score":0.6107994914054871},{"id":"https://openalex.org/keywords/learning-classifier-system","display_name":"Learning classifier system","score":0.5904116630554199},{"id":"https://openalex.org/keywords/leverage","display_name":"Leverage (statistics)","score":0.5698514580726624},{"id":"https://openalex.org/keywords/computational-learning-theory","display_name":"Computational learning theory","score":0.5003113746643066},{"id":"https://openalex.org/keywords/categorization","display_name":"Categorization","score":0.46246689558029175},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.4588106572628021},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.44509491324424744},{"id":"https://openalex.org/keywords/q-learning","display_name":"Q-learning","score":0.4408456087112427},{"id":"https://openalex.org/keywords/unsupervised-learning","display_name":"Unsupervised learning","score":0.4314422011375427},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.41887426376342773}],"concepts":[{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.798820972442627},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7622866630554199},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.7387844920158386},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7370942234992981},{"id":"https://openalex.org/C24138899","wikidata":"https://www.wikidata.org/wiki/Q17141258","display_name":"Instance-based learning","level":3,"score":0.6107994914054871},{"id":"https://openalex.org/C199190896","wikidata":"https://www.wikidata.org/wiki/Q3509276","display_name":"Learning classifier system","level":3,"score":0.5904116630554199},{"id":"https://openalex.org/C153083717","wikidata":"https://www.wikidata.org/wiki/Q6535263","display_name":"Leverage (statistics)","level":2,"score":0.5698514580726624},{"id":"https://openalex.org/C50292564","wikidata":"https://www.wikidata.org/wiki/Q2462783","display_name":"Computational learning theory","level":3,"score":0.5003113746643066},{"id":"https://openalex.org/C94124525","wikidata":"https://www.wikidata.org/wiki/Q912550","display_name":"Categorization","level":2,"score":0.46246689558029175},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.4588106572628021},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.44509491324424744},{"id":"https://openalex.org/C188116033","wikidata":"https://www.wikidata.org/wiki/Q2664563","display_name":"Q-learning","level":3,"score":0.4408456087112427},{"id":"https://openalex.org/C8038995","wikidata":"https://www.wikidata.org/wiki/Q1152135","display_name":"Unsupervised learning","level":2,"score":0.4314422011375427},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.41887426376342773}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.1109/access.2019.2941229","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2941229","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08836506.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},{"id":"pmh:oai:doaj.org/article:3650edfa2a4a4dde9d25ad7c8abd4584","is_oa":true,"landing_page_url":"https://doaj.org/article/3650edfa2a4a4dde9d25ad7c8abd4584","pdf_url":null,"source":{"id":"https://openalex.org/S4306401280","display_name":"DOAJ (DOAJ: Directory of Open Access Journals)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-sa","license_id":"https://openalex.org/licenses/cc-by-sa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"IEEE Access, Vol 7, Pp 133653-133667 (2019)","raw_type":"article"}],"best_oa_location":{"id":"doi:10.1109/access.2019.2941229","is_oa":true,"landing_page_url":"https://doi.org/10.1109/access.2019.2941229","pdf_url":"https://ieeexplore.ieee.org/ielx7/6287639/8600701/08836506.pdf","source":{"id":"https://openalex.org/S2485537415","display_name":"IEEE Access","issn_l":"2169-3536","issn":["2169-3536"],"is_oa":true,"is_in_doaj":true,"is_core":true,"host_organization":"https://openalex.org/P4310319808","host_organization_name":"Institute of Electrical and Electronics Engineers","host_organization_lineage":["https://openalex.org/P4310319808"],"host_organization_lineage_names":["Institute of Electrical and Electronics Engineers"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"IEEE Access","raw_type":"journal-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":210,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W98006832","https://openalex.org/W126344362","https://openalex.org/W158722652","https://openalex.org/W180325379","https://openalex.org/W1496327573","https://openalex.org/W1505515950","https://openalex.org/W1531886263","https://openalex.org/W1537934887","https://openalex.org/W1540725368","https://openalex.org/W1546167776","https://openalex.org/W1553414138","https://openalex.org/W1566652554","https://openalex.org/W1582436621","https://openalex.org/W1591803298","https://openalex.org/W1592847719","https://openalex.org/W1640646391","https://openalex.org/W1735161956","https://openalex.org/W1757796397","https://openalex.org/W1845972764","https://openalex.org/W1901616594","https://openalex.org/W1964463172","https://openalex.org/W1973039793","https://openalex.org/W1976473913","https://openalex.org/W1977655452","https://openalex.org/W1985031440","https://openalex.org/W1989400239","https://openalex.org/W2000875519","https://openalex.org/W2002256627","https://openalex.org/W2056325504","https://openalex.org/W2095487261","https://openalex.org/W2096630263","https://openalex.org/W2103145033","https://openalex.org/W2105643086","https://openalex.org/W2110431754","https://openalex.org/W2115192345","https://openalex.org/W2115211925","https://openalex.org/W2116551132","https://openalex.org/W2117316169","https://openalex.org/W2118686230","https://openalex.org/W2119567691","https://openalex.org/W2120846115","https://openalex.org/W2121517924","https://openalex.org/W2121615981","https://openalex.org/W2124175081","https://openalex.org/W2126226307","https://openalex.org/W2128569883","https://openalex.org/W2130416410","https://openalex.org/W2131600418","https://openalex.org/W2132351269","https://openalex.org/W2137564278","https://openalex.org/W2138362680","https://openalex.org/W2145339207","https://openalex.org/W2145952809","https://openalex.org/W2147062835","https://openalex.org/W2147750403","https://openalex.org/W2149986801","https://openalex.org/W2154535830","https://openalex.org/W2155027007","https://openalex.org/W2155968351","https://openalex.org/W2162729077","https://openalex.org/W2165501837","https://openalex.org/W2172193467","https://openalex.org/W2173248099","https://openalex.org/W2181252817","https://openalex.org/W2189058185","https://openalex.org/W2202743478","https://openalex.org/W2212473182","https://openalex.org/W2236244207","https://openalex.org/W2250340166","https://openalex.org/W2258956268","https://openalex.org/W2281071090","https://openalex.org/W2293184044","https://openalex.org/W2334782222","https://openalex.org/W2395575420","https://openalex.org/W2440926996","https://openalex.org/W2519114575","https://openalex.org/W2553188523","https://openalex.org/W2555906998","https://openalex.org/W2565313327","https://openalex.org/W2575705757","https://openalex.org/W2579592432","https://openalex.org/W2592798481","https://openalex.org/W2594794854","https://openalex.org/W2602963933","https://openalex.org/W2604550840","https://openalex.org/W2604735539","https://openalex.org/W2606101940","https://openalex.org/W2610686804","https://openalex.org/W2614309179","https://openalex.org/W2617411258","https://openalex.org/W2618749399","https://openalex.org/W2620974420","https://openalex.org/W2624731731","https://openalex.org/W2625614184","https://openalex.org/W2626804490","https://openalex.org/W2683470288","https://openalex.org/W2730274928","https://openalex.org/W2733312032","https://openalex.org/W2738778707","https://openalex.org/W2741381403","https://openalex.org/W2742079690","https://openalex.org/W2746553466","https://openalex.org/W2746722165","https://openalex.org/W2753088790","https://openalex.org/W2754187158","https://openalex.org/W2754930636","https://openalex.org/W2762709466","https://openalex.org/W2763208138","https://openalex.org/W2765990695","https://openalex.org/W2771590529","https://openalex.org/W2772589676","https://openalex.org/W2774145422","https://openalex.org/W2775468524","https://openalex.org/W2780057514","https://openalex.org/W2782108898","https://openalex.org/W2786414509","https://openalex.org/W2788862220","https://openalex.org/W2808421695","https://openalex.org/W2920925306","https://openalex.org/W2941568898","https://openalex.org/W2949267040","https://openalex.org/W2950892788","https://openalex.org/W2951799221","https://openalex.org/W2962710194","https://openalex.org/W2962717849","https://openalex.org/W2962887844","https://openalex.org/W2963000099","https://openalex.org/W2963111827","https://openalex.org/W2963300197","https://openalex.org/W2963376229","https://openalex.org/W2963676620","https://openalex.org/W2963797805","https://openalex.org/W2963864421","https://openalex.org/W2963866658","https://openalex.org/W2963943581","https://openalex.org/W2964251366","https://openalex.org/W2964345382","https://openalex.org/W3098163860","https://openalex.org/W3100789280","https://openalex.org/W3100857292","https://openalex.org/W3106462682","https://openalex.org/W4233696721","https://openalex.org/W4247008852","https://openalex.org/W4253572110","https://openalex.org/W4285719527","https://openalex.org/W4293872189","https://openalex.org/W4297692223","https://openalex.org/W4298023569","https://openalex.org/W4298188144","https://openalex.org/W4298857966","https://openalex.org/W4300427714","https://openalex.org/W4300997991","https://openalex.org/W4321392130","https://openalex.org/W6604066176","https://openalex.org/W6605172240","https://openalex.org/W6630047662","https://openalex.org/W6632089484","https://openalex.org/W6632293826","https://openalex.org/W6632625769","https://openalex.org/W6634907821","https://openalex.org/W6635327058","https://openalex.org/W6636932949","https://openalex.org/W6637967152","https://openalex.org/W6676576769","https://openalex.org/W6677067356","https://openalex.org/W6678114464","https://openalex.org/W6678168664","https://openalex.org/W6678317597","https://openalex.org/W6680197816","https://openalex.org/W6681825124","https://openalex.org/W6682329378","https://openalex.org/W6683204974","https://openalex.org/W6684921986","https://openalex.org/W6685285391","https://openalex.org/W6685366099","https://openalex.org/W6685444567","https://openalex.org/W6686898673","https://openalex.org/W6688425321","https://openalex.org/W6691553443","https://openalex.org/W6696891444","https://openalex.org/W6712181171","https://openalex.org/W6718190810","https://openalex.org/W6726982070","https://openalex.org/W6729964979","https://openalex.org/W6731613598","https://openalex.org/W6734215269","https://openalex.org/W6734678876","https://openalex.org/W6735011893","https://openalex.org/W6735677848","https://openalex.org/W6735768275","https://openalex.org/W6736534893","https://openalex.org/W6737210826","https://openalex.org/W6738144653","https://openalex.org/W6738383168","https://openalex.org/W6739112683","https://openalex.org/W6739585900","https://openalex.org/W6742058293","https://openalex.org/W6743684492","https://openalex.org/W6744298676","https://openalex.org/W6744361522","https://openalex.org/W6745120743","https://openalex.org/W6745649294","https://openalex.org/W6746352052","https://openalex.org/W6746825387","https://openalex.org/W6747359475","https://openalex.org/W6748599296","https://openalex.org/W6748645729","https://openalex.org/W6748910126","https://openalex.org/W6760349497"],"related_works":["https://openalex.org/W4286799911","https://openalex.org/W3196155444","https://openalex.org/W3140225428","https://openalex.org/W4205569898","https://openalex.org/W2610686804","https://openalex.org/W4310801741","https://openalex.org/W3042291365","https://openalex.org/W4321844043","https://openalex.org/W2171968955","https://openalex.org/W2171128163"],"abstract_inverted_index":{"Q-learning":[0,59,107,110,133,160,179],"is":[1,38],"arguably":[2],"one":[3,13],"of":[4,14,21,71,101,106,150],"the":[5,15,19,88,93,98,137],"most":[6],"applied":[7,124],"representative":[8],"reinforcement":[9,30,147],"learning":[10,31,115,148],"approaches":[11],"and":[12,32,51,66,86,123,157,164,174],"off-policy":[16],"strategies.":[17],"Since":[18],"emergence":[20],"Q-learning,":[22],"many":[23],"studies":[24],"have":[25,120],"described":[26],"its":[27,144],"uses":[28],"in":[29,63],"artificial":[33,55],"intelligence":[34,56],"problems.":[35],"However,":[36],"there":[37],"an":[39],"information":[40],"gap":[41],"as":[42,142],"to":[43],"how":[44,132],"these":[45],"powerful":[46,82],"algorithms":[47,60,161],"can":[48],"be":[49],"leveraged":[50],"incorporated":[52],"into":[53,162],"general":[54,99],"workflow.":[57],"Early":[58],"were":[61],"unsatisfactory":[62],"several":[64],"aspects":[65],"covered":[67],"a":[68],"narrow":[69],"range":[70],"applications.":[72],"It":[73],"has":[74],"also":[75],"been":[76,121],"observed":[77],"that":[78,177],"sometimes,":[79],"this":[80,127],"rather":[81],"algorithm":[83],"learns":[84],"unrealistically":[85],"overestimates":[87],"action":[89],"values":[90],"hence":[91],"abating":[92],"overall":[94],"performance.":[95],"Recently":[96],"with":[97,116],"advances":[100],"machine":[102],"learning,":[103],"more":[104],"variants":[105,153],"like":[108],"Deep":[109],"which":[111],"combines":[112],"basic":[113],"Q":[114],"deep":[117],"neural":[118],"networks":[119],"discovered":[122],"extensively.":[125],"In":[126],"paper,":[128],"we":[129,158,168],"thoroughly":[130,169],"explain":[131],"evolved":[134],"by":[135],"unraveling":[136],"mathematical":[138],"complexities":[139],"behind":[140],"it":[141],"well":[143],"flow":[145],"from":[146],"family":[149],"algorithms.":[151,180],"Improved":[152],"are":[154],"fully":[155],"described,":[156],"categorize":[159],"single-agent":[163],"multi-agent":[165],"approaches.":[166],"Finally,":[167],"investigate":[170],"up-to-date":[171],"research":[172],"trends":[173],"key":[175],"applications":[176],"leverage":[178]},"counts_by_year":[{"year":2026,"cited_by_count":53},{"year":2025,"cited_by_count":160},{"year":2024,"cited_by_count":166},{"year":2023,"cited_by_count":105},{"year":2022,"cited_by_count":86},{"year":2021,"cited_by_count":41},{"year":2020,"cited_by_count":12},{"year":2019,"cited_by_count":1}],"updated_date":"2026-07-30T17:31:21.811387","created_date":"2025-10-10T00:00:00"}