{"id":"https://openalex.org/W3173515307","doi":"https://doi.org/10.1007/s10107-022-01920-6","title":"Softmax policy gradient methods can take exponential time to converge","display_name":"Softmax policy gradient methods can take exponential time to converge","publication_year":2023,"publication_date":"2023-01-23","ids":{"openalex":"https://openalex.org/W3173515307","doi":"https://doi.org/10.1007/s10107-022-01920-6","mag":"3173515307"},"language":"en","primary_location":{"id":"doi:10.1007/s10107-022-01920-6","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10107-022-01920-6","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10107-022-01920-6.pdf","source":{"id":"https://openalex.org/S193920097","display_name":"Mathematical Programming","issn_l":"0025-5610","issn":["0025-5610","1436-4646"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Mathematical Programming","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"hybrid","oa_url":"https://link.springer.com/content/pdf/10.1007/s10107-022-01920-6.pdf","any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100357037","display_name":"Gen Li","orcid":"https://orcid.org/0000-0002-6883-2407"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Gen Li","raw_affiliation_strings":["Department of Statistics and Data Science, The Wharton School, University of Pennsylvania, Philadelphia, PA, 19104, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science, The Wharton School, University of Pennsylvania, Philadelphia, PA, 19104, USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102736633","display_name":"Yuting Wei","orcid":null},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuting Wei","raw_affiliation_strings":["Department of Statistics and Data Science, The Wharton School, University of Pennsylvania, Philadelphia, PA, 19104, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science, The Wharton School, University of Pennsylvania, Philadelphia, PA, 19104, USA","institution_ids":["https://openalex.org/I79576946"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053809095","display_name":"Yuejie Chi","orcid":"https://orcid.org/0000-0002-6766-5459"},"institutions":[{"id":"https://openalex.org/I74973139","display_name":"Carnegie Mellon University","ror":"https://ror.org/05x2bcf33","country_code":"US","type":"education","lineage":["https://openalex.org/I74973139"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Yuejie Chi","raw_affiliation_strings":["Department of Electrical and Computer Engineering, Carnegie Mellon University, Pittsburgh, PA, 15213, USA"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Electrical and Computer Engineering, Carnegie Mellon University, Pittsburgh, PA, 15213, USA","institution_ids":["https://openalex.org/I74973139"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100416078","display_name":"Yuxin Chen","orcid":"https://orcid.org/0000-0001-9256-5815"},"institutions":[{"id":"https://openalex.org/I79576946","display_name":"University of Pennsylvania","ror":"https://ror.org/00b30xv10","country_code":"US","type":"education","lineage":["https://openalex.org/I79576946"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Yuxin Chen","raw_affiliation_strings":["Department of Statistics and Data Science, The Wharton School, University of Pennsylvania, Philadelphia, PA, 19104, USA"],"raw_orcid":"https://orcid.org/0000-0001-9256-5815","affiliations":[{"raw_affiliation_string":"Department of Statistics and Data Science, The Wharton School, University of Pennsylvania, Philadelphia, PA, 19104, USA","institution_ids":["https://openalex.org/I79576946"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5100416078"],"corresponding_institution_ids":["https://openalex.org/I79576946"],"apc_list":{"value":2890,"currency":"USD","value_usd":2890},"apc_paid":{"value":2890,"currency":"USD","value_usd":2890},"fwci":0.8455,"has_fulltext":true,"cited_by_count":9,"citation_normalized_percentile":{"value":0.75987122,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":94,"max":98},"biblio":{"volume":"201","issue":"1-2","first_page":"707","last_page":"802"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998999834060669,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9861999750137329,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10502","display_name":"Advanced Memory and Neural Computing","score":0.9850000143051147,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.9061534404754639},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.6252644658088684},{"id":"https://openalex.org/keywords/cardinality","display_name":"Cardinality (data modeling)","score":0.5519770383834839},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.525024950504303},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.4270175099372864},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.4010748565196991},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3614017367362976},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.34651172161102295},{"id":"https://openalex.org/keywords/database","display_name":"Database","score":0.25266504287719727},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.0901276171207428}],"concepts":[{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.9061534404754639},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.6252644658088684},{"id":"https://openalex.org/C87117476","wikidata":"https://www.wikidata.org/wiki/Q362383","display_name":"Cardinality (data modeling)","level":2,"score":0.5519770383834839},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.525024950504303},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4270175099372864},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.4010748565196991},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3614017367362976},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.34651172161102295},{"id":"https://openalex.org/C77088390","wikidata":"https://www.wikidata.org/wiki/Q8513","display_name":"Database","level":1,"score":0.25266504287719727},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.0901276171207428},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1007/s10107-022-01920-6","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10107-022-01920-6","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10107-022-01920-6.pdf","source":{"id":"https://openalex.org/S193920097","display_name":"Mathematical Programming","issn_l":"0025-5610","issn":["0025-5610","1436-4646"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Mathematical Programming","raw_type":"journal-article"}],"best_oa_location":{"id":"doi:10.1007/s10107-022-01920-6","is_oa":true,"landing_page_url":"https://doi.org/10.1007/s10107-022-01920-6","pdf_url":"https://link.springer.com/content/pdf/10.1007/s10107-022-01920-6.pdf","source":{"id":"https://openalex.org/S193920097","display_name":"Mathematical Programming","issn_l":"0025-5610","issn":["0025-5610","1436-4646"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310319900","host_organization_name":"Springer Science+Business Media","host_organization_lineage":["https://openalex.org/P4310319900","https://openalex.org/P4310319965"],"host_organization_lineage_names":["Springer Science+Business Media","Springer Nature"],"type":"journal"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Mathematical Programming","raw_type":"journal-article"},"sustainable_development_goals":[{"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions","score":0.800000011920929}],"awards":[{"id":"https://openalex.org/G1056225131","display_name":null,"funder_award_id":"FA9550-22-1-0198","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G1469985387","display_name":null,"funder_award_id":"N00014-19-1-2404","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G1523888516","display_name":null,"funder_award_id":"FA9550-","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G3024259446","display_name":"CAREER: Statistical Learning from a Modern Perspective: Over-parameterization, Regularization, and Generalization","funder_award_id":"2143215","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3467583470","display_name":null,"funder_award_id":"N00014-19-1-2120","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G3550625438","display_name":"Collaborative Research: CIF: Medium: Statistical and Algorithmic Foundations of Efficient Reinforcement Learning","funder_award_id":"2221009","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G3582032785","display_name":null,"funder_award_id":"CCF-2007911","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4061742820","display_name":null,"funder_award_id":"FA9550-19-1-0030","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G4558904681","display_name":null,"funder_award_id":"W911NF-20-1-0097","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G4720003262","display_name":null,"funder_award_id":"N00014-22","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G4776652967","display_name":"RI: Small: Uncertainty Quantification for  Nonconvex Low-Complexity Models","funder_award_id":"2218773","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4910588663","display_name":"Collaborative Research: CIF: Medium: Statistical and Algorithmic Foundations of Efficient Reinforcement Learning","funder_award_id":"2106739","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5174315691","display_name":"Collaborative Research: CIF: Medium: Statistical and Algorithmic Foundations of Efficient Reinforcement Learning","funder_award_id":"2106778","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G5359226925","display_name":"CIF: Small: Taming Nonconvexity in High-Dimensional Statistical Estimation","funder_award_id":"1907661","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G556410599","display_name":null,"funder_award_id":"W911NF-18-1-0303","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G5809100787","display_name":null,"funder_award_id":"FA9550","funder_id":"https://openalex.org/F4320338279","funder_display_name":"Air Force Office of Scientific Research"},{"id":"https://openalex.org/G5945795592","display_name":null,"funder_award_id":"CCF-1907661","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6098521345","display_name":null,"funder_award_id":"FA9550","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G6636086349","display_name":"CIF: Small: Resource-Efficient Statistical Inference in Networked Environments","funder_award_id":"2007911","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G7452299184","display_name":null,"funder_award_id":"W911NF","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G8195561347","display_name":"RI: Medium: Collaborative Research:Algorithmic High-Dimensional Statistics: Optimality, Computtional Barriers, and High-Dimensional Corrections","funder_award_id":"2218713","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8272153555","display_name":"CIF: Medium: Collaborative Research: Nonconvex Optimization for High-Dimensional Signal Estimation: Theory and Fast Algorithms","funder_award_id":"1806154","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G854152385","display_name":null,"funder_award_id":"N00014-18-1-2142","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"},{"id":"https://openalex.org/G8597684915","display_name":null,"funder_award_id":"W911NF-18-1-0303","funder_id":"https://openalex.org/F4320338281","funder_display_name":"Army Research Office"},{"id":"https://openalex.org/G8876996369","display_name":null,"funder_award_id":"N00014","funder_id":"https://openalex.org/F4320337345","funder_display_name":"Office of Naval Research"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"},{"id":"https://openalex.org/F4320338279","display_name":"Air Force Office of Scientific Research","ror":"https://ror.org/011e9bt93"},{"id":"https://openalex.org/F4320338281","display_name":"Army Research Office","ror":"https://ror.org/05epdh915"}],"has_content":{"grobid_xml":false,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3173515307.pdf"},"referenced_works_count":24,"referenced_works":["https://openalex.org/W569478347","https://openalex.org/W2120678009","https://openalex.org/W2145339207","https://openalex.org/W2172968643","https://openalex.org/W2257979135","https://openalex.org/W2763081248","https://openalex.org/W2998050631","https://openalex.org/W3046017196","https://openalex.org/W3090442079","https://openalex.org/W3093528669","https://openalex.org/W3109546547","https://openalex.org/W3124786193","https://openalex.org/W3127686539","https://openalex.org/W3203759272","https://openalex.org/W3209135762","https://openalex.org/W3217314940","https://openalex.org/W6600127060","https://openalex.org/W6600134738","https://openalex.org/W6600165366","https://openalex.org/W6600175266","https://openalex.org/W6600708292","https://openalex.org/W6814003322","https://openalex.org/W6816678993","https://openalex.org/W6854636774"],"related_works":["https://openalex.org/W3107204728","https://openalex.org/W4287591324","https://openalex.org/W4226420367","https://openalex.org/W2980176872","https://openalex.org/W2962876041","https://openalex.org/W3090555870","https://openalex.org/W3108503355","https://openalex.org/W2249953602","https://openalex.org/W2801655600","https://openalex.org/W2912971006"],"abstract_inverted_index":{"Abstract":[0],"The":[1],"softmax":[2,12,53,117,132],"policy":[3,13,24,166],"gradient":[4,9,125],"(PG)":[5],"method,":[6],"which":[7,99],"performs":[8],"ascent":[10],"under":[11],"parameterization,":[14],"is":[15,182,187],"arguably":[16],"one":[17],"of":[18,23,52,66,70,80,98,116,163,210],"the":[19,78,81,89,113,131,161,178,191,208],"de":[20],"facto":[21],"implementations":[22],"optimization":[25],"in":[26,56,160,219],"modern":[27],"reinforcement":[28],"learning.":[29],"For":[30],"$$\\gamma":[31],"$$":[32,138],"<mml:math":[33,86,94,139,155],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"><mml:mi>\u03b3</mml:mi></mml:math>":[34],"-discounted":[35],"infinite-horizon":[36],"tabular":[37],"Markov":[38],"decision":[39],"processes":[40],"(MDPs),":[41],"remarkable":[42],"progress":[43],"has":[44],"recently":[45],"been":[46],"achieved":[47],"towards":[48],"establishing":[49],"global":[50],"convergence":[51,71],"PG":[54,118,133,221],"methods":[55],"finding":[57],"a":[58,109,164,195],"near-optimal":[59],"policy.":[60],"However,":[61],"prior":[62],"results":[63],"fall":[64],"short":[65],"delineating":[67],"clear":[68],"dependencies":[69],"rates":[72],"on":[73],"salient":[74],"parameters":[75],"such":[76],"as":[77],"cardinality":[79],"state":[82,171],"space":[83],"$${\\mathcal":[84],"{S}}$$":[85],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"><mml:mi>S</mml:mi></mml:math>":[87],"and":[88,168],"effective":[90],"horizon":[91],"$$\\frac{1}{1-\\gamma":[92],"}$$":[93],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"><mml:mfrac><mml:mn>1</mml:mn><mml:mrow><mml:mn>1</mml:mn><mml:mo>-</mml:mo><mml:mi>\u03b3</mml:mi></mml:mrow></mml:mfrac></mml:math>":[95],",":[96],"both":[97],"could":[100],"be":[101],"excessively":[102],"large.":[103],"In":[104],"this":[105],"paper,":[106],"we":[107,128],"deliver":[108],"pessimistic":[110],"message":[111],"regarding":[112],"iteration":[114],"complexity":[115],"methods,":[119],"despite":[120],"assuming":[121],"access":[122],"to":[123,157,174],"exact":[124],"computation.":[126],"Specifically,":[127],"demonstrate":[129],"that":[130,177],"method":[134],"with":[135],"stepsize":[136],"$$\\eta":[137],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"><mml:mi>\u03b7</mml:mi></mml:math>":[140],"can":[141],"take":[142],"$$\\begin{aligned}":[143],"\\frac{1}{\\eta":[144],"}":[145],"|{\\mathcal":[146],"{S}}|^{2^{\\Omega":[147],"\\big":[148],"(\\frac{1}{1-\\gamma":[149],"}\\big":[150],")}}":[151],"~\\text":[152],"{iterations}":[153],"\\end{aligned}$$":[154],"xmlns:mml=\"http://www.w3.org/1998/Math/MathML\"><mml:mrow><mml:mtable><mml:mtr><mml:mtd><mml:mrow><mml:mfrac><mml:mn>1</mml:mn><mml:mi>\u03b7</mml:mi></mml:mfrac><mml:msup><mml:mrow><mml:mo>|</mml:mo><mml:mi>S</mml:mi><mml:mo>|</mml:mo></mml:mrow><mml:msup><mml:mn>2</mml:mn><mml:mrow><mml:mi>\u03a9</mml:mi><mml:mrow><mml:mo>(</mml:mo></mml:mrow><mml:mfrac><mml:mn>1</mml:mn><mml:mrow><mml:mn>1</mml:mn><mml:mo>-</mml:mo><mml:mi>\u03b3</mml:mi></mml:mrow></mml:mfrac><mml:mrow><mml:mo>)</mml:mo></mml:mrow></mml:mrow></mml:msup></mml:msup><mml:mspace/><mml:mtext>iterations</mml:mtext></mml:mrow></mml:mtd></mml:mtr></mml:mtable></mml:mrow></mml:math>":[156],"converge,":[158],"even":[159],"presence":[162],"benign":[165],"initialization":[167],"an":[169],"initial":[170],"distribution":[172,179],"amenable":[173],"exploration":[175],"(so":[176],"mismatch":[180],"coefficient":[181],"not":[183],"exceedingly":[184],"large).":[185],"This":[186],"accomplished":[188],"by":[189],"characterizing":[190],"algorithmic":[192],"dynamics":[193],"over":[194],"carefully-constructed":[196],"MDP":[197],"containing":[198],"only":[199],"three":[200],"actions.":[201],"Our":[202],"exponential":[203],"lower":[204],"bound":[205],"hints":[206],"at":[207],"necessity":[209],"carefully":[211],"adjusting":[212],"update":[213],"rules":[214],"or":[215],"enforcing":[216],"proper":[217],"regularization":[218],"accelerating":[220],"methods.":[222]},"counts_by_year":[{"year":2026,"cited_by_count":2},{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2022,"cited_by_count":3}],"updated_date":"2026-07-23T08:03:31.855105","created_date":"2025-10-10T00:00:00"}