{"id":"https://openalex.org/W4384203266","doi":"https://doi.org/10.1137/22m149185x","title":"Dual Ascent and Primal-Dual Algorithms for Infinite-Horizon Nonstationary Markov Decision Processes","display_name":"Dual Ascent and Primal-Dual Algorithms for Infinite-Horizon Nonstationary Markov Decision Processes","publication_year":2023,"publication_date":"2023-07-13","ids":{"openalex":"https://openalex.org/W4384203266","doi":"https://doi.org/10.1137/22m149185x"},"language":"en","primary_location":{"id":"doi:10.1137/22m149185x","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1137/22m149185x","pdf_url":null,"source":{"id":"https://openalex.org/S928796702","display_name":"SIAM Journal on Optimization","issn_l":"1052-6234","issn":["1052-6234","1095-7189"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Optimization","raw_type":"journal-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068691426","display_name":"Archis Ghate","orcid":"https://orcid.org/0000-0001-6093-5340"},"institutions":[{"id":"https://openalex.org/I201448701","display_name":"University of Washington","ror":"https://ror.org/00cvxb145","country_code":"US","type":"education","lineage":["https://openalex.org/I201448701"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Archis Ghate","raw_affiliation_strings":["Department of Industrial & Systems Engineering, University of Washington, Seattle, WA 98195 USA"],"affiliations":[{"raw_affiliation_string":"Department of Industrial & Systems Engineering, University of Washington, Seattle, WA 98195 USA","institution_ids":["https://openalex.org/I201448701"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5068691426"],"corresponding_institution_ids":["https://openalex.org/I201448701"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.08803222,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":"33","issue":"3","first_page":"1391","last_page":"1415"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T13553","display_name":"Age of Information Optimization","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T13553","display_name":"Age of Information Optimization","score":0.9915000200271606,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12288","display_name":"Optimization and Search Problems","score":0.9837999939918518,"subfield":{"id":"https://openalex.org/subfields/1705","display_name":"Computer Networks and Communications"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10974","display_name":"Advanced Queuing Theory Analysis","score":0.9722999930381775,"subfield":{"id":"https://openalex.org/subfields/1404","display_name":"Management Information Systems"},"field":{"id":"https://openalex.org/fields/14","display_name":"Business, Management and Accounting"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.7823477983474731},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.6706286668777466},{"id":"https://openalex.org/keywords/monotonic-function","display_name":"Monotonic function","score":0.6472390294075012},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.6282191872596741},{"id":"https://openalex.org/keywords/dual","display_name":"Dual (grammatical number)","score":0.6201461553573608},{"id":"https://openalex.org/keywords/sequence","display_name":"Sequence (biology)","score":0.6179285645484924},{"id":"https://openalex.org/keywords/duality","display_name":"Duality (order theory)","score":0.5846368670463562},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.5759283304214478},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5408068895339966},{"id":"https://openalex.org/keywords/duality-gap","display_name":"Duality gap","score":0.5239061117172241},{"id":"https://openalex.org/keywords/dynamic-programming","display_name":"Dynamic programming","score":0.48241767287254333},{"id":"https://openalex.org/keywords/rate-of-convergence","display_name":"Rate of convergence","score":0.4535863697528839},{"id":"https://openalex.org/keywords/optimal-stopping","display_name":"Optimal stopping","score":0.4256798326969147},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.3441586196422577},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.33780616521835327},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.28496256470680237},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.2672141194343567},{"id":"https://openalex.org/keywords/discrete-mathematics","display_name":"Discrete mathematics","score":0.08612862229347229}],"concepts":[{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.7823477983474731},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.6706286668777466},{"id":"https://openalex.org/C72169020","wikidata":"https://www.wikidata.org/wiki/Q194404","display_name":"Monotonic function","level":2,"score":0.6472390294075012},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.6282191872596741},{"id":"https://openalex.org/C2780980858","wikidata":"https://www.wikidata.org/wiki/Q110022","display_name":"Dual (grammatical number)","level":2,"score":0.6201461553573608},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.6179285645484924},{"id":"https://openalex.org/C2778023678","wikidata":"https://www.wikidata.org/wiki/Q554403","display_name":"Duality (order theory)","level":2,"score":0.5846368670463562},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.5759283304214478},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5408068895339966},{"id":"https://openalex.org/C5274546","wikidata":"https://www.wikidata.org/wiki/Q5310264","display_name":"Duality gap","level":3,"score":0.5239061117172241},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.48241767287254333},{"id":"https://openalex.org/C57869625","wikidata":"https://www.wikidata.org/wiki/Q1783502","display_name":"Rate of convergence","level":3,"score":0.4535863697528839},{"id":"https://openalex.org/C99414536","wikidata":"https://www.wikidata.org/wiki/Q7098950","display_name":"Optimal stopping","level":2,"score":0.4256798326969147},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.3441586196422577},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.33780616521835327},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.28496256470680237},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.2672141194343567},{"id":"https://openalex.org/C118615104","wikidata":"https://www.wikidata.org/wiki/Q121416","display_name":"Discrete mathematics","level":1,"score":0.08612862229347229},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C54355233","wikidata":"https://www.wikidata.org/wiki/Q7162","display_name":"Genetics","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C124952713","wikidata":"https://www.wikidata.org/wiki/Q8242","display_name":"Literature","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C142362112","wikidata":"https://www.wikidata.org/wiki/Q735","display_name":"Art","level":0,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1137/22m149185x","is_oa":false,"landing_page_url":"http://dx.doi.org/10.1137/22m149185x","pdf_url":null,"source":{"id":"https://openalex.org/S928796702","display_name":"SIAM Journal on Optimization","issn_l":"1052-6234","issn":["1052-6234","1095-7189"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310320508","host_organization_name":"Society for Industrial and Applied Mathematics","host_organization_lineage":["https://openalex.org/P4310320508"],"host_organization_lineage_names":["Society for Industrial and Applied Mathematics"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"SIAM Journal on Optimization","raw_type":"journal-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","score":0.8100000023841858,"id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":34,"referenced_works":["https://openalex.org/W133224232","https://openalex.org/W892566935","https://openalex.org/W1521018645","https://openalex.org/W1762430620","https://openalex.org/W1966091678","https://openalex.org/W1970986504","https://openalex.org/W1977191384","https://openalex.org/W1982928247","https://openalex.org/W1984901446","https://openalex.org/W1989567750","https://openalex.org/W2000132205","https://openalex.org/W2002492712","https://openalex.org/W2019857351","https://openalex.org/W2035320001","https://openalex.org/W2048543781","https://openalex.org/W2051994935","https://openalex.org/W2062213626","https://openalex.org/W2073103075","https://openalex.org/W2105260964","https://openalex.org/W2112865659","https://openalex.org/W2114019863","https://openalex.org/W2116387273","https://openalex.org/W2147867446","https://openalex.org/W2162748558","https://openalex.org/W2334782222","https://openalex.org/W2399915317","https://openalex.org/W2582186538","https://openalex.org/W2883274460","https://openalex.org/W2964034900","https://openalex.org/W3007764701","https://openalex.org/W3122502719","https://openalex.org/W4205808827","https://openalex.org/W4255867662","https://openalex.org/W4298856952"],"related_works":["https://openalex.org/W4255265352","https://openalex.org/W4239477580","https://openalex.org/W4389475841","https://openalex.org/W2952594763","https://openalex.org/W4285537323","https://openalex.org/W2903299703","https://openalex.org/W4281791088","https://openalex.org/W4385342861","https://openalex.org/W2117282672","https://openalex.org/W1574958246"],"abstract_inverted_index":{"Infinite-horizon":[0],"nonstationary":[1],"Markov":[2],"decision":[3],"processes":[4],"(MDPs)":[5],"extend":[6],"their":[7],"stationary":[8],"counterparts":[9],"by":[10],"allowing":[11],"temporal":[12],"variations":[13],"in":[14,58,117],"immediate":[15],"costs":[16],"and":[17,24,40,88,113,166,202,223,238],"transition":[18],"probabilities.":[19],"Bellman\u2019s":[20,118,200],"characterization":[21],"of":[22,38,55,66,152,194,233,243],"optimality":[23,90],"equivalent":[25],"primal-dual":[26,182,234,240],"linear":[27],"programming":[28],"formulations":[29],"for":[30],"these":[31,60,101,185],"MDPs":[32],"include":[33],"a":[34,48,64,150,161,189,211,218],"countably":[35],"infinite":[36],"number":[37],"variables":[39,122,135],"equations.":[41,119],"Simple":[42],"policy":[43,83,248],"iteration,":[44],"also":[45,224],"viewed":[46],"as":[47],"primal":[49,190,225],"simplex":[50],"algorithm,":[51],"is":[52,85,92,210,241],"the":[53,56,174,181,197,231],"state":[54],"art":[57],"solving":[59],"MDPs.":[61],"It":[62,146,187],"produces":[63],"sequence":[65,151],"policies":[67,153],"whose":[68],"costs-to-go":[69,110],"converge":[70,136],"monotonically":[71,137],"from":[72,78,138],"above":[73],"to":[74,99,140,217],"optimal.":[75,141],"This":[76,131,142],"suffers":[77],"two":[79,96,144],"limitations.":[80,145,186],"A":[81],"cost-improving":[82],"update":[84],"computationally":[86],"expensive":[87],"an":[89,170],"gap":[91],"missing.":[93],"We":[94],"propose":[95],"dual-based":[97],"approaches":[98],"address":[100],"concerns.":[102],"The":[103,120,177,208],"first,":[104],"called":[105],"dual":[106,121,134,192,205,236],"ascent,":[107],"maintains":[108,188],"approximate":[109],"(dual":[111],"variables)":[112],"corresponding":[114,198],"nonnegative":[115],"errors":[116,128],"are":[123],"iteratively":[124],"increased":[125],"such":[126],"that":[127,133,239],"vanish":[129],"asymptotically.":[130],"guarantees":[132],"below":[139],"has":[143],"does":[147,158,167],"not":[148,159,168],"maintain":[149],"(primal":[154],"variables).":[155],"Hence,":[156],"it":[157],"provide":[160],"decision-making":[162],"strategy":[163],"at":[164],"termination":[165],"offer":[169],"upper":[171],"bound":[172],"on":[173],"optimal":[175],"costs-to-go.":[176],"second":[178],"approach,":[179],"termed":[180],"method,":[183],"addresses":[184],"policy,":[191],"approximations":[193],"its":[195],"costs-to-go,":[196],"nonegative":[199],"errors,":[201],"inherits":[203],"monotonic":[204],"value":[206,226],"convergence.":[207,227],"key":[209],"so-called":[212],"rebalancing":[213],"step,":[214],"which":[215],"leads":[216],"duality":[219],"gap\u2013based":[220],"stopping":[221],"criterion":[222],"Computational":[228],"experiments":[229],"demonstrate":[230],"benefits":[232],"over":[235],"ascent":[237],"orders":[242],"magnitude":[244],"faster":[245],"than":[246],"simple":[247],"iteration.":[249]},"counts_by_year":[],"updated_date":"2026-02-24T19:35:01.260952","created_date":"2025-10-10T00:00:00"}