{"id":"https://openalex.org/W7147586800","doi":"https://doi.org/10.1145/3797823.3797849","title":"When Q-Learning fails: unstable behavior for infinite state spaces","display_name":"When Q-Learning fails: unstable behavior for infinite state spaces","publication_year":2026,"publication_date":"2026-03-31","ids":{"openalex":"https://openalex.org/W7147586800","doi":"https://doi.org/10.1145/3797823.3797849"},"language":"en","primary_location":{"id":"doi:10.1145/3797823.3797849","doi":"https://doi.org/10.1145/3797823.3797849","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3797823.3797849","pdf_url":null,"source":{"id":"https://openalex.org/S4210187660","display_name":"ACM SIGMETRICS Performance Evaluation Review","issn_l":"0163-5999","issn":["0163-5999","1557-9484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGMETRICS Performance Evaluation Review","raw_type":"journal-article"},"volume":null,"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5132640694","display_name":"U. Ayestaa","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I4210119061","display_name":"Institut de Recherche en Informatique de Toulouse","ror":"https://ror.org/01rx4qw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I205747304","https://openalex.org/I4210119061","https://openalex.org/I4210152422","https://openalex.org/I4405258862","https://openalex.org/I4405259414"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"U. Ayestaa","raw_affiliation_strings":["CNRS, IRIT, 2 rue C. Camichel,  31071 Toulouse, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CNRS, IRIT, 2 rue C. Camichel,  31071 Toulouse, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4210119061"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061112228","display_name":"Sergey Foss","orcid":"https://orcid.org/0000-0003-0116-5846","observed_orcids":["https://orcid.org/0000-0003-0116-5846"]},"institutions":[{"id":"https://openalex.org/I32062511","display_name":"Heriot-Watt University","ror":"https://ror.org/04mghma93","country_code":"GB","type":"education","lineage":["https://openalex.org/I32062511"]},{"id":"https://openalex.org/I4210117017","display_name":"Maxwell Institute for Mathematical Sciences","ror":"https://ror.org/02tsqtg57","country_code":"GB","type":"facility","lineage":["https://openalex.org/I32062511","https://openalex.org/I4210117017","https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"S. Foss","raw_affiliation_strings":["School of MACS, Heriot-Watt University and Maxwell Institute, EH14 4AS, Edinburgh, United Kingdom"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"School of MACS, Heriot-Watt University and Maxwell Institute, EH14 4AS, Edinburgh, United Kingdom","institution_ids":["https://openalex.org/I32062511","https://openalex.org/I4210117017"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053044735","display_name":"Matthieu Jonckheere","orcid":"https://orcid.org/0000-0003-3576-5866","observed_orcids":["https://orcid.org/0000-0003-3576-5866"]},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"M. Jonckheere","raw_affiliation_strings":["CNRS, LAAS, 7 av. du colonel Roche, 31031 Toulouse, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CNRS, LAAS, 7 av. du colonel Roche, 31031 Toulouse, France","institution_ids":["https://openalex.org/I1294671590"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5132725888","display_name":"V. Puricellia","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I4210119061","display_name":"Institut de Recherche en Informatique de Toulouse","ror":"https://ror.org/01rx4qw44","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I205747304","https://openalex.org/I4210119061","https://openalex.org/I4210152422","https://openalex.org/I4405258862","https://openalex.org/I4405259414"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"V. Puricellia","raw_affiliation_strings":["CNRS, IRIT, 2 rue C. Camichel, 31071 Toulouse, France"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CNRS, IRIT, 2 rue C. Camichel, 31071 Toulouse, France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4210119061"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.21187295,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"53","issue":"4","first_page":"79","last_page":"83"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8306000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.8306000232696533,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10974","display_name":"Advanced Queuing Theory Analysis","score":0.11659999936819077,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.010700000450015068,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/q-learning","display_name":"Q-learning","score":0.9990000128746033},{"id":"https://openalex.org/keywords/infinite-state-spaces","display_name":"infinite state spaces","score":0.8659999966621399},{"id":"https://openalex.org/keywords/convergence-failure","display_name":"convergence failure","score":0.5379999876022339},{"id":"https://openalex.org/keywords/stability","display_name":"stability","score":0.48500001430511475},{"id":"https://openalex.org/keywords/transient-response","display_name":"transient response","score":0.7429999709129333},{"id":"https://openalex.org/keywords/model-free-reinforcement-learning","display_name":"model-free reinforcement learning","score":0.7910000085830688},{"id":"https://openalex.org/keywords/load-balancing","display_name":"load balancing","score":0.7739999890327454},{"id":"https://openalex.org/keywords/queueing-models","display_name":"queueing models","score":0.8410000205039978},{"id":"https://openalex.org/keywords/countably-infinite-state-space","display_name":"countably infinite state space","score":0.6269999742507935}],"concepts":[{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6741999983787537},{"id":"https://openalex.org/C110729354","wikidata":"https://www.wikidata.org/wiki/Q185478","display_name":"Countable set","level":2,"score":0.6498000025749207},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.618399977684021},{"id":"https://openalex.org/C160403385","wikidata":"https://www.wikidata.org/wiki/Q220543","display_name":"Queue","level":2,"score":0.5322999954223633},{"id":"https://openalex.org/C22684755","wikidata":"https://www.wikidata.org/wiki/Q847526","display_name":"Queueing theory","level":2,"score":0.5192000269889832},{"id":"https://openalex.org/C48103436","wikidata":"https://www.wikidata.org/wiki/Q599031","display_name":"State (computer science)","level":2,"score":0.5047000050544739},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.5037000179290771},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.451200008392334},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.43700000643730164},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.41429999470710754},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.4138999879360199},{"id":"https://openalex.org/C2780799671","wikidata":"https://www.wikidata.org/wiki/Q17087362","display_name":"Transient (computer programming)","level":2,"score":0.40799999237060547},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.3578000068664551},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.3474999964237213},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.33070001006126404},{"id":"https://openalex.org/C8171440","wikidata":"https://www.wikidata.org/wiki/Q903414","display_name":"Steady state (chemistry)","level":2,"score":0.3059999942779541},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.2957000136375427},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.29120001196861267},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.28690001368522644},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.2842000126838684},{"id":"https://openalex.org/C57945734","wikidata":"https://www.wikidata.org/wiki/Q7977941","display_name":"Weak convergence","level":3,"score":0.28049999475479126},{"id":"https://openalex.org/C191399111","wikidata":"https://www.wikidata.org/wiki/Q64861","display_name":"Transitive relation","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C91575142","wikidata":"https://www.wikidata.org/wiki/Q1971426","display_name":"Optimal control","level":2,"score":0.26570001244544983},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2628999948501587},{"id":"https://openalex.org/C33891772","wikidata":"https://www.wikidata.org/wiki/Q6505536","display_name":"Layered queueing network","level":3,"score":0.25920000672340393},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.257999986410141},{"id":"https://openalex.org/C207821765","wikidata":"https://www.wikidata.org/wiki/Q405372","display_name":"Instability","level":2,"score":0.25690001249313354}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1145/3797823.3797849","doi":"https://doi.org/10.1145/3797823.3797849","is_oa":false,"landing_page_url":"https://doi.org/10.1145/3797823.3797849","pdf_url":null,"source":{"id":"https://openalex.org/S4210187660","display_name":"ACM SIGMETRICS Performance Evaluation Review","issn_l":"0163-5999","issn":["0163-5999","1557-9484"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-1"],"host_organization":"https://openalex.org/P4310319798","host_organization_name":"Association for Computing Machinery","host_organization_lineage":["https://openalex.org/P4310319798"],"host_organization_lineage_names":["Association for Computing Machinery"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ACM SIGMETRICS Performance Evaluation Review","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[],"sustainable_development_goals_aurora":[{"display_name":"Decent work and economic growth","id":"https://metadata.un.org/sdg/8","score":0.6079878807067871}],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":7,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W1967141673","https://openalex.org/W1999254175","https://openalex.org/W2147750403","https://openalex.org/W2334782222","https://openalex.org/W3047127288","https://openalex.org/W4413132625"],"related_works":[],"abstract_inverted_index":{"The":[0,79],"Q-learning":[1,101],"algorithm":[2],"is":[3],"well":[4],"known":[5],"for":[6],"its":[7,22],"convergence":[8],"guarantees":[9],"to":[10,65,83,106,108],"the":[11,87,109,119,134,143],"optimal":[12,110],"policy":[13],"in":[14,24,33,118,127],"finite-state":[15],"environments.":[16],"In":[17,57],"this":[18,37,58],"paper,":[19],"we":[20,39],"investigate":[21],"limitations":[23],"countable":[25],"infinite":[26,54,128],"state":[27,55],"spaces":[28],"\u2013":[29],"a":[30,41,47,52,60,115],"setting":[31],"common":[32],"real-world":[34],"problems.":[35],"To":[36],"end,":[38],"introduce":[40],"simple":[42],"queueing":[43],"model,":[44,59],"based":[45],"on":[46],"load":[48],"balancing":[49],"problem,":[50],"with":[51,138],"countably":[53],"space.":[56],"dispatcher":[61],"assigns":[62],"incoming":[63],"jobs":[64],"one":[66],"of":[67,122],"two":[68,73],"queues":[69],"by":[70],"choosing":[71],"between":[72],"possible":[74],"actions:":[75],"''red''":[76,80],"and":[77,104],"''green''.":[78],"action":[81,89],"leads":[82],"transient":[84],"behavior,":[85],"whereas":[86],"''green''":[88],"ensures":[90],"stability.":[91],"Our":[92,112],"main":[93],"result":[94],"shows":[95],"that,":[96],"under":[97],"certain":[98],"parameter":[99],"conditions,":[100],"exhibits":[102],"instability":[103],"fails":[105],"converge":[107],"policy.":[111],"findings":[113],"reveal":[114],"critical":[116],"gap":[117],"theoretical":[120],"understanding":[121],"model-free":[123],"Reinforcement":[124],"Learning":[125],"methods":[126],"domains.":[129],"Numerical":[130],"experiments":[131],"illustrate":[132],"that":[133,141],"transience":[135],"also":[136],"occurs":[137],"decreasing":[139],"stepsizes":[140],"satisfy":[142],"usual":[144],"Robbins-Monro":[145],"conditions.":[146]},"counts_by_year":[],"updated_date":"2026-09-21T07:43:02.447757","created_date":"2026-04-02T00:00:00"}