{"id":"https://openalex.org/W4413132625","doi":"https://doi.org/10.1007/s11134-025-09949-y","title":"Convergence of Natural Policy Gradient for a family of infinite-state queueing MDPs","display_name":"Convergence of Natural Policy Gradient for a family of infinite-state queueing MDPs","publication_year":2025,"publication_date":"2025-08-07","ids":{"openalex":"https://openalex.org/W4413132625","doi":"https://doi.org/10.1007/s11134-025-09949-y"},"language":"en","primary_location":{"id":"doi:10.1007/s11134-025-09949-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11134-025-09949-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11134-025-09949-y.pdf","source":{"id":"https://openalex.org/S148503350","display_name":"Queueing Systems","issn_l":"0257-0130","issn":["0257-0130","1572-9443"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Queueing Systems","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s11134-025-09949-y.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5001892692","display_name":"Isaac Grosof","orcid":"https://orcid.org/0000-0001-6205-8652"},"institutions":[{"id":"https://openalex.org/I111979921","display_name":"Northwestern University","ror":"https://ror.org/000e0be47","country_code":"US","type":"education","lineage":["https://openalex.org/I111979921"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Isaac Grosof","raw_affiliation_strings":["Department of Industrial Engineering and Management Science, Northwestern University, Evanston, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Industrial Engineering and Management Science, Northwestern University, Evanston, IL, USA","institution_ids":["https://openalex.org/I111979921"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021806638","display_name":"Siva Theja Maguluri","orcid":"https://orcid.org/0000-0002-5797-1639"},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Siva Theja Maguluri","raw_affiliation_strings":["School of Industrial and Systems Engineering, Georgia Institute of Technology, Atlanta, GA, USA"],"affiliations":[{"raw_affiliation_string":"School of Industrial and Systems Engineering, Georgia Institute of Technology, Atlanta, GA, USA","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5078518595","display_name":"R. Srikant","orcid":"https://orcid.org/0000-0003-1483-5204"},"institutions":[{"id":"https://openalex.org/I157725225","display_name":"University of Illinois Urbana-Champaign","ror":"https://ror.org/047426m28","country_code":"US","type":"education","lineage":["https://openalex.org/I157725225"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"R. Srikant","raw_affiliation_strings":["Department of Electrical and Computer Engineering, University of Illinois, Urbana-Champaign, Urbana, IL, USA"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, University of Illinois, Urbana-Champaign, Urbana, IL, USA","institution_ids":["https://openalex.org/I157725225"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":3,"corresponding_author_ids":["https://openalex.org/A5001892692"],"corresponding_institution_ids":["https://openalex.org/I111979921"],"apc_list":{"value":2290,"currency":"EUR","value_usd":2890},"apc_paid":{"value":2290,"currency":"EUR","value_usd":2890},"fwci":4.1324,"has_fulltext":true,"cited_by_count":2,"citation_normalized_percentile":{"value":0.94257447,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":91,"max":99},"biblio":{"volume":"109","issue":"3","first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9995999932289124,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9988999962806702,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13553","display_name":"Age of Information Optimization","score":0.995199978351593,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.7325741052627563},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7060667276382446},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.6791762113571167},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.5803706049919128},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.521007776260376},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.4503551423549652},{"id":"https://openalex.org/keywords/queueing-theory","display_name":"Queueing theory","score":0.4499725103378296},{"id":"https://openalex.org/keywords/variety","display_name":"Variety (cybernetics)","score":0.41553473472595215},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.32988572120666504},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.31631484627723694},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.23413413763046265},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.19943389296531677},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.16946330666542053},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.06054490804672241}],"concepts":[{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.7325741052627563},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7060667276382446},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.6791762113571167},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.5803706049919128},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.521007776260376},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.4503551423549652},{"id":"https://openalex.org/C22684755","wikidata":"https://www.wikidata.org/wiki/Q847526","display_name":"Queueing theory","level":2,"score":0.4499725103378296},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.41553473472595215},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.32988572120666504},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.31631484627723694},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.23413413763046265},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19943389296531677},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.16946330666542053},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.06054490804672241},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C31258907","wikidata":"https://www.wikidata.org/wiki/Q1301371","display_name":"Computer network","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s11134-025-09949-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11134-025-09949-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11134-025-09949-y.pdf","source":{"id":"https://openalex.org/S148503350","display_name":"Queueing Systems","issn_l":"0257-0130","issn":["0257-0130","1572-9443"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Queueing Systems","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s11134-025-09949-y","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s11134-025-09949-y","pdf_url":"https://link.springer.com/content/pdf/10.1007/s11134-025-09949-y.pdf","source":{"id":"https://openalex.org/S148503350","display_name":"Queueing Systems","issn_l":"0257-0130","issn":["0257-0130","1572-9443"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Queueing Systems","raw_type":"journal-article"},"sustainable_development_goals":[{"score":0.800000011920929,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[{"id":"https://openalex.org/G1104240869","display_name":null,"funder_award_id":"FA9550-24-1-0002","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G1523888516","display_name":null,"funder_award_id":"FA9550-","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G216924667","display_name":null,"funder_award_id":"CNS 23-12714","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4085306640","display_name":null,"funder_award_id":"CCF 22-07547","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5809100787","display_name":null,"funder_award_id":"FA9550","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G6298352139","display_name":null,"funder_award_id":"CNS 21-06801","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6894402473","display_name":null,"funder_award_id":"Fellowship","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7152741550","display_name":null,"funder_award_id":"EPCN-2144316","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G735048306","display_name":"CAREER: Lyapunov Drift Methods for Stochastic Recursions: Applications in Cloud Computing and Reinforcement Learning","funder_award_id":"2144316","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8452575566","display_name":null,"funder_award_id":"CPS-2240982","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G848032724","display_name":null,"funder_award_id":"Science","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320309475","display_name":"Northwestern University","ror":"https://ror.org/000e0be47"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4413132625.pdf","grobid_xml":"https://content.openalex.org/works/W4413132625.grobid-xml"},"referenced_works_count":25,"referenced_works":["https://openalex.org/W9378068","https://openalex.org/W639949082","https://openalex.org/W1497026160","https://openalex.org/W1591180426","https://openalex.org/W1616818660","https://openalex.org/W1974518363","https://openalex.org/W1979675141","https://openalex.org/W1999102639","https://openalex.org/W2057938070","https://openalex.org/W2074680702","https://openalex.org/W2081320273","https://openalex.org/W2155770202","https://openalex.org/W2172968643","https://openalex.org/W2565612488","https://openalex.org/W2948432982","https://openalex.org/W2981743241","https://openalex.org/W2990138404","https://openalex.org/W2999905431","https://openalex.org/W3047127288","https://openalex.org/W4214717370","https://openalex.org/W4247230948","https://openalex.org/W4288447305","https://openalex.org/W4391020645","https://openalex.org/W6631189298","https://openalex.org/W6814003322"],"related_works":["https://openalex.org/W4400868993","https://openalex.org/W3096874164","https://openalex.org/W1985560493","https://openalex.org/W2386410636","https://openalex.org/W2357975469","https://openalex.org/W2145363145","https://openalex.org/W1626977535","https://openalex.org/W4284974072","https://openalex.org/W2341346307","https://openalex.org/W4404955202"],"abstract_inverted_index":{"Abstract":[0],"A":[1],"wide":[2],"variety":[3,24],"of":[4,25,39,82,96,147,194],"queueing":[5,97,159],"systems":[6],"can":[7],"be":[8],"naturally":[9],"modeled":[10],"as":[11,46],"infinite-state":[12,148],"Markov":[13],"Decision":[14],"Processes":[15],"(MDPs).":[16],"In":[17],"the":[18,37,53,72,80,83,119,123,129,134,140,158,185,191,195],"reinforcement":[19],"learning":[20,43],"(RL)":[21],"context,":[22],"a":[23,93,101,144,156,172],"algorithms":[26,66],"have":[27],"been":[28],"developed":[29],"to":[30,88,155,161,178],"learn":[31],"and":[32,50,99],"optimize":[33],"these":[34,64],"MDPs.":[35,150],"At":[36],"heart":[38],"many":[40],"popular":[41],"policy-gradient-based":[42],"algorithms,":[44],"such":[45],"natural":[47],"actor-critic,":[48],"TRPO,":[49],"PPO,":[51],"lies":[52],"Natural":[54],"Policy":[55],"Gradient":[56],"(NPG)":[57],"policy":[58],"optimization":[59],"algorithm.":[60,74,197],"Convergence":[61],"results":[62,70,78],"for":[63,71,118,139,143],"RL":[65],"rest":[67],"on":[68,79,184],"convergence":[69,81,116,136],"NPG":[73,84,120,124,141,196],"However,":[75],"all":[76],"existing":[77],"algorithm":[85,125,142],"are":[86,181],"limited":[87],"finite-state":[89],"settings.":[90],"We":[91],"study":[92],"general":[94,145],"class":[95,146],"MDPs":[98],"prove":[100],"$$O(1/\\sqrt{T})$$":[102],"<mml:math":[103],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\">":[104],"<mml:mrow>":[105],"<mml:mi>O</mml:mi>":[106],"<mml:mo>(</mml:mo>":[107],"<mml:mn>1</mml:mn>":[108],"<mml:mo>/</mml:mo>":[109],"<mml:msqrt>":[110],"<mml:mi>T</mml:mi>":[111],"</mml:msqrt>":[112],"<mml:mo>)</mml:mo>":[113],"</mml:mrow>":[114],"</mml:math>":[115],"rate":[117,137],"algorithm,":[121],"if":[122],"is":[126,133],"initialized":[127],"with":[128],"MaxWeight":[130],"policy.":[131,176],"This":[132],"first":[135],"bound":[138],"average-reward":[149],"Moreover,":[151],"our":[152,179],"result":[153,180],"applies":[154],"beyond":[157],"setting":[160],"any":[162],"countably":[163],"infinite":[164],"MDP":[165],"satisfying":[166],"certain":[167],"mild":[168],"structural":[169],"assumptions,":[170],"given":[171],"sufficiently":[173],"good":[174],"initial":[175],"Key":[177],"state-dependent":[182],"bounds":[183],"relative":[186],"value":[187],"function":[188],"achieved":[189],"by":[190],"iterate":[192],"policies":[193]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":1}],"updated_date":"2026-04-02T15:55:50.835912","created_date":"2025-10-10T00:00:00"}