{"id":"https://openalex.org/W2074261021","doi":"https://doi.org/10.7148/2009-0537-0543","title":"Simulation-Based Optimization Of Markov Controlled Processes With Unknown Parameters","display_name":"Simulation-Based Optimization Of Markov Controlled Processes With Unknown Parameters","publication_year":2009,"publication_date":"2009-06-09","ids":{"openalex":"https://openalex.org/W2074261021","doi":"https://doi.org/10.7148/2009-0537-0543","mag":"2074261021"},"language":"en","primary_location":{"id":"doi:10.7148/2009-0537-0543","is_oa":false,"landing_page_url":"https://doi.org/10.7148/2009-0537-0543","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ECMS 2009 Proceedings edited by J. Otamendi, A. Bargiela, J. L. Montes, L. M. Doncel Pedrera","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5004457099","display_name":"Enrique Campos-N\u00e1\u00f1ez","orcid":"https://orcid.org/0000-0002-3036-3772"},"institutions":[{"id":"https://openalex.org/I193531525","display_name":"George Washington University","ror":"https://ror.org/00y4zzh67","country_code":"US","type":"education","lineage":["https://openalex.org/I193531525"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Enrique Campos-Nanez","raw_affiliation_strings":["george Washington University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"george Washington University","institution_ids":["https://openalex.org/I193531525"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5004457099"],"corresponding_institution_ids":["https://openalex.org/I193531525"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.16075157,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"537","last_page":"543"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T14082","display_name":"Modeling, Simulation, and Optimization","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2607","display_name":"Discrete Mathematics and Combinatorics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T14082","display_name":"Modeling, Simulation, and Optimization","score":0.9986000061035156,"subfield":{"id":"https://openalex.org/subfields/2607","display_name":"Discrete Mathematics and Combinatorics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11801","display_name":"Reservoir Engineering and Simulation Methods","score":0.9894000291824341,"subfield":{"id":"https://openalex.org/subfields/2212","display_name":"Ocean Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.9703999757766724,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.6656376719474792},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6506868600845337},{"id":"https://openalex.org/keywords/curse-of-dimensionality","display_name":"Curse of dimensionality","score":0.5654985904693604},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.5154119729995728},{"id":"https://openalex.org/keywords/convergence","display_name":"Convergence (economics)","score":0.5052855610847473},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.4706764817237854},{"id":"https://openalex.org/keywords/dynamic-programming","display_name":"Dynamic programming","score":0.46280166506767273},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.45630282163619995},{"id":"https://openalex.org/keywords/local-optimum","display_name":"Local optimum","score":0.44886529445648193},{"id":"https://openalex.org/keywords/parameterized-complexity","display_name":"Parameterized complexity","score":0.41593965888023376},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3644055128097534},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.235042542219162},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.13710379600524902}],"concepts":[{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.6656376719474792},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6506868600845337},{"id":"https://openalex.org/C111030470","wikidata":"https://www.wikidata.org/wiki/Q1430460","display_name":"Curse of dimensionality","level":2,"score":0.5654985904693604},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.5154119729995728},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.5052855610847473},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.4706764817237854},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.46280166506767273},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.45630282163619995},{"id":"https://openalex.org/C141934464","wikidata":"https://www.wikidata.org/wiki/Q3305386","display_name":"Local optimum","level":2,"score":0.44886529445648193},{"id":"https://openalex.org/C165464430","wikidata":"https://www.wikidata.org/wiki/Q1570441","display_name":"Parameterized complexity","level":2,"score":0.41593965888023376},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3644055128097534},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.235042542219162},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.13710379600524902},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C50522688","wikidata":"https://www.wikidata.org/wiki/Q189833","display_name":"Economic growth","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.7148/2009-0537-0543","is_oa":false,"landing_page_url":"https://doi.org/10.7148/2009-0537-0543","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"ECMS 2009 Proceedings edited by J. Otamendi, A. Bargiela, J. L. Montes, L. M. Doncel Pedrera","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.4099999964237213,"id":"https://metadata.un.org/sdg/8","display_name":"Decent work and economic growth"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":25,"referenced_works":["https://openalex.org/W1554366315","https://openalex.org/W1576452626","https://openalex.org/W1604128822","https://openalex.org/W1968375151","https://openalex.org/W2003308588","https://openalex.org/W2009797711","https://openalex.org/W2027045220","https://openalex.org/W2068782949","https://openalex.org/W2078273556","https://openalex.org/W2097415784","https://openalex.org/W2104830490","https://openalex.org/W2115597380","https://openalex.org/W2120465407","https://openalex.org/W2122715318","https://openalex.org/W2123225686","https://openalex.org/W2123826923","https://openalex.org/W2132291040","https://openalex.org/W2134437265","https://openalex.org/W2143116909","https://openalex.org/W2159871982","https://openalex.org/W2171033527","https://openalex.org/W2320680700","https://openalex.org/W2329331109","https://openalex.org/W2531891978","https://openalex.org/W3103182070"],"related_works":["https://openalex.org/W2049472152","https://openalex.org/W4389475841","https://openalex.org/W4241327272","https://openalex.org/W2120406836","https://openalex.org/W1996214847","https://openalex.org/W2903299703","https://openalex.org/W2117282672","https://openalex.org/W4211224558","https://openalex.org/W4281791088","https://openalex.org/W3208659968"],"abstract_inverted_index":{"We":[0,14,35,53,104,492,544,603,622,676,828,888],"consider":[1,15,481],"simulation-based":[2,572],"gradient-estimation":[3],"and":[4,28,43,101,136,177,195,200,204,208,212,255,259,342,346,375,405,409,445,471,500,537,565,580,612,640,663,673,704,707,770,823,856,872],"its":[5,727],"use":[6,38,116,169,216,630],"in":[7,99,175,187,230,235,338,373,383,403,424,435,507,555,633,671,712,719,797,812],"Markov":[8,17,318,489,508,610,746,795],"controlled":[9,20],"processes":[10,221,510],"with":[11,108,474,625,748,838],"unknown":[12,542,620,774],"parameters.":[13,130],"a":[16,23,29,39,83,109,231,331,384,414,439,464,532,539,594,626,744,761,771,779,851,878],"reward":[18,490,505,509,521,844,862,895],"process":[19,382],"by":[21,81,280,289,297,846,885],"both":[22],"set":[24,30,431,533,762],"of":[25,31,58,76,121,155,170,185,218,239,283,300,306,314,317,325,330,334,484,487,497,502,526,529,534,560,563,578,589,608,616,618,631,683,723,763,783,788,793,818,853,880],"tunable":[26,535],"parameters,":[27,303],"fixed":[32,540,772],"but":[33,262,296,358,427,541,773],"unknown.":[34],"analyze":[36],"the":[37,74,96,119,122,147,153,168,181,219,240,284,304,312,328,361,380,396,429,450,458,482,495,503,513,519,556,561,576,587,590,606,614,629,681,684,692,720,724,791,794,798,801,819,824,834,860,866,893],"recursive":[40,85],"identification":[41],"procedure,":[42],"their":[44],"application":[45,94],"to":[46,78,95,117,125,159,167,183,222,268,293,432,437,570,593,687,832,892],"existing":[47,571],"gradient-based":[48,232],"algorithms":[49,241],"based":[50],"on":[51,638],"simulation.":[52],"show":[54],"that":[55,215,226,360,416,548,800],"simple":[56],"modifications":[57],"available":[59],"gradient":[60,224,270],"estimation":[61,381,499,546,617],"algorithms,":[62,573,662],"namely":[63],"assuming":[64],"parameter":[65,70,695,775,825],"certainty,":[66],"can":[67,139,227,277,367,460,549,708,898],"accommodate":[68],"system":[69,129,285,302,421,694,848,867],"identification,":[71],"without":[72],"sacrificing":[73],"convergence":[75,266,588,682],"these":[77,161],"local":[79,595,688],"optima":[80],"following":[82],"two-time-scale":[84],"identification/optimization":[86],"procedure.":[87],"This":[88,275],"approach":[89,666],"is":[90,309,391,417,423,599,717,733,760,849,868,877],"illustrated":[91],"through":[92],"an":[93,619,668,713],"algorithm":[97,150,577,669,700],"proposed":[98,123],"(Marbach":[100,176,194,579,672],"Tsitsiklis,":[102,178,196,472,581,674],"2001).":[103,675],"illustrate":[105,664],"our":[106,665,890],"results":[107,562],"small":[110,702],"numerical":[111],"example,":[112],"which":[113,390,512,897],"we":[114,480],"further":[115],"test":[118],"ability":[120],"scheme":[124,686],"track":[126],"slow":[127,265,730],"changing":[128,731],"INTRODUCTION":[131],"Many":[132,238],"stochastic":[133,220,252],"resource":[134],"allocation":[135],"control":[137,172,316,820],"problems":[138],"be":[140,228,278,368,433,461,550,710,743,833,899],"naturally":[141],"modeled":[142],"using":[143,667],"dynamic":[144,148],"programming.":[145],"Unfortunately,":[146],"programming":[149],"suffers":[151],"from":[152],"\u2018curse":[154],"dimensionality\u2019.":[156],"In":[157,454,477],"order":[158,436],"address":[160,327,613],"performance":[162,716],"problems,":[163],"researchers":[164],"have":[165],"recurred":[166],"parameterized":[171,488],"policies,":[173],"as":[174,337,372,449,516,518,575,601,901],"2001),":[179,582],"reducing":[180],"problem":[182,276,313,483,496,607,615],"one":[184],"search":[186,233],"policy":[188,236],"space.":[189,237],"Several":[190],"simulationbased":[191],"techniques":[192,308],"exist":[193],"2001,":[197],"2003;":[198],"Cao":[199,203],"Chen,":[201],"1997;":[202],"Wan,":[205],"1998;":[206],"Fang":[207],"Cao,":[209],"2004;":[210],"Baxter":[211],"Bartlett,":[213],"2001)":[214],"realizations":[217],"obtain":[223],"estimates":[225,412],"used":[229],"method":[234,592],"mentioned":[242],"above":[243],"are":[244,399,524,816,875],"\u2018modelfree\u2019":[245],"and,":[246],"hence,":[247],"implicitly":[248],"adaptive,":[249],"e.g.,":[250],"general":[251],"approximations":[253],"(Kiefer":[254],"Wolfowitz,":[256],"1952;":[257],"Kushner":[258],"Yin,":[260],"1997),":[261],"they":[263],"exhibit":[264],"due":[267],"biased":[269],"estimates,":[271],"or":[272,467],"large":[273],"variances.":[274],"alleviated":[279],"incorporating":[281],"knowledge":[282,299],"model,":[286],"for":[287,364,441,511,586,680],"example":[288],"exploiting":[290],"regenerative":[291],"structure":[292],"eliminate":[294],"bias,":[295],"requiring":[298],"all":[301],"usefulness":[305],"such":[307,371,574],"limited.":[310],"Although":[311],"adaptive":[315,418,485,609,685,699],"chains":[319],"has":[320,701],"been":[321],"studied":[322],"before,":[323],"most":[324],"authors":[326],"case":[329],"finite":[332,434,749],"number":[333],"possible":[335],"models,":[336],"(Mandl,":[339],"1974;":[340],"Borkar":[341],"Varaiya,":[343,376],"1979;":[344],"Kumar":[345],"Becker,":[347],"1982;":[348],"Doshi,":[349],"1980;":[350],"El-Fattah,":[351],"1981).":[352],"Other":[353],"authors,":[354],"overcome":[355],"this":[356,478,624,847],"restriction,":[357],"assume":[359],"optimal":[362],"policies":[363],"each":[365,442,553],"model":[366,422],"easily":[369],"computed,":[370],"(Borkar":[374],"1982).":[377],"Others,":[378],"embed":[379],"value":[385],"iteration":[386],"procedure":[387],"(Hernandez-Lerma,":[388],"1989),":[389],"not":[392],"efficient":[393],"computationally":[394],"when":[395,865],"state":[397,451,750,870],"spaces":[398],"large.":[400],"The":[401,597,697,843],"work":[402],"(Ren":[404],"Krogh,":[406],"2001;":[407],"Santharam":[408],"Sastry,":[410],"1997)":[411],"Q-factors,":[413],"technique":[415],"since":[419],"no":[420],"principle":[425],"needed,":[426],"requires":[428],"action":[430],"build":[438],"Q-factor":[440],"state-action":[443],"pair,":[444],"quickly":[446],"becomes":[447],"impractical":[448],"space":[452,751],"increases.":[453],"some":[455,559],"specific":[456],"cases,":[457],"Q-factors":[459],"approximated":[462],"via":[463],"neural":[465],"network":[466],"other":[468],"architectures":[469],"(Bertsekas":[470],"1996),":[473],"limited":[475],"success.":[476],"paper":[479,598],"optimization":[486,501,661],"processes.":[491],"first":[493,604],"study":[494,545],"online":[498],"average":[504,894],"criterion":[506],"transition":[514,554,802,835],"probabilities,":[515],"well":[517],"expected":[520,861],"per":[522,863],"stage,":[523],"functions":[525,817],"two":[527,786],"sets":[528,787],"parameters:":[530],"1)":[531],"controls,":[536],"2)":[538],"parameter.":[543,621],"procedures":[547],"updated":[551],"after":[552],"chain,":[557],"generalizing":[558],"(Campos-Nanez":[564],"Patek,":[566],"2005),":[567],"apply":[568],"them":[569],"providing":[583],"sufficient":[584,678],"conditions":[585,679],"resulting":[591,698],"optimum.":[596],"organized":[600],"follows.":[602],"present":[605],"control,":[611],"follow":[623],"discussion":[627],"about":[628],"estimators":[632],"Proceedings":[634],"23rd":[635],"European":[636],"Conference":[637],"Modelling":[639],"Simulation":[641],"\u00a9ECMS":[642],"Javier":[643],"Otamendi,":[644],"Andrzej":[645],"Bargiela,":[646],"Jose":[647],"Luis":[648,650],"Montes,":[649],"Miguel":[651],"Doncel":[652],"Pedrera":[653],"(Editors)":[654],"ISBN:":[655,658],"978-0-9553018-8-9":[656],"/":[657],"978-0-9553018-9-6":[659],"(CD)":[660],"developed":[670],"provide":[677],"optima,":[689],"while":[690],"identifying":[691],"true":[693],"values.":[696],"memory":[703],"computational":[705],"requirements,":[706],"hence":[709],"implemented":[711],"fashion.":[714],"Its":[715],"discussed":[718],"last":[721],"section":[722],"paper,":[725],"where":[726],"usability":[728],"under":[729],"parameters":[732,765,789,854],"explored.":[734],"MARKOV":[735],"REWARD":[736],"PROCESSES":[737],"WITH":[738],"UNKNOWN":[739],"PARAMETERS":[740],"Let":[741],"{in}n":[742],"discrete-time":[745],"chain":[747,796],"S":[752],"=":[753,807,809,813,905],"{1,2,":[754],".":[755,756,757],",N}.":[758],"There":[759],"\u2018tunable\u2019":[764],"u":[766,874],"\u2208":[767],"R":[768],",":[769],"\u03b8":[776,826,857,882],"\u2217":[777],"\u2208\u0398,":[778],"convex,":[780],"compact":[781],"subset":[782],"R.":[784],"These":[785],"determine":[790],"dynamics":[792],"sense":[799],"probabilities":[803],"pi":[804,840],"j(u,\u03b8":[805,841],"\u2217)":[806,831,904,916],"P(in+1":[808],"j":[810],"|":[811],"i,u,\u03b8":[814],"\u2217),":[815],"vector":[821],"u,":[822,855,881],"\u2217.":[827],"define":[829],"P(u,\u03b8":[830],"probability":[836],"matrix":[837],"entries":[839],"\u2217).":[842,887],"observed":[845],"also":[850],"function":[852,879],"\u2217,":[858,883],"i.e.,":[859],"stage":[864],"at":[869],"i,":[871],"controls":[873],"applied":[876],"denoted":[884],"gi(u,\u03b8":[886],"focus":[889],"attention":[891],"criterion,":[896],"defined":[900],"\u03bb":[902],"(u,\u03b8":[903],"liminf":[906],"T\u2192\u221e":[907],"1":[908],"T":[909,912],"E":[910],"[":[911],"\u2211":[913],"n=0":[914],"gin(u,\u03b8":[915],"\u2223":[917]},"counts_by_year":[],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}