{"id":"https://openalex.org/W1557287189","doi":"https://doi.org/10.1109/cdc.2003.1272362","title":"Implementation of gradient estimation to a constrained Markov decision problem","display_name":"Implementation of gradient estimation to a constrained Markov decision problem","publication_year":2004,"publication_date":"2004-06-21","ids":{"openalex":"https://openalex.org/W1557287189","doi":"https://doi.org/10.1109/cdc.2003.1272362","mag":"1557287189"},"language":"en","primary_location":{"id":"doi:10.1109/cdc.2003.1272362","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc.2003.1272362","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"42nd IEEE International Conference on Decision and Control (IEEE Cat. No.03CH37475)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5068804090","display_name":"Vikram Krishnamurthy","orcid":"https://orcid.org/0000-0002-4170-6056"},"institutions":[{"id":"https://openalex.org/I141945490","display_name":"University of British Columbia","ror":"https://ror.org/03rmrcq20","country_code":"CA","type":"education","lineage":["https://openalex.org/I141945490"]},{"id":"https://openalex.org/I165779595","display_name":"The University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU","CA"],"is_corresponding":false,"raw_author_name":"V. Krishnamurthy","raw_affiliation_strings":["Depanment of Electrical and Electronic Engineering, University of Melbourne, VIC, Australia","Department of Electrical and Computer Engineering, University of British Columbia, Vancouver, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Depanment of Electrical and Electronic Engineering, University of Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I165779595"]},{"raw_affiliation_string":"Department of Electrical and Computer Engineering, University of British Columbia, Vancouver, Canada","institution_ids":["https://openalex.org/I141945490"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032264786","display_name":"Kipp Martin","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"K. Martin","raw_affiliation_strings":["D\u00e9partement dinformatique et recherche op\u00e9rationnelte, Universit\u00e9 de Montreal, Quebec, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"D\u00e9partement dinformatique et recherche op\u00e9rationnelte, Universit\u00e9 de Montreal, Quebec, Canada","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5025478222","display_name":"FJ Abad","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"F. Vasquez Abad","raw_affiliation_strings":["D\u00e9partement dinformatique et recherche op\u00e9rationnelte, Universit\u00e9 de Montreal, Quebec, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"D\u00e9partement dinformatique et recherche op\u00e9rationnelte, Universit\u00e9 de Montreal, Quebec, Canada","institution_ids":["https://openalex.org/I70931966"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":2.313,"has_fulltext":false,"cited_by_count":22,"citation_normalized_percentile":{"value":0.89081886,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":97},"biblio":{"volume":null,"issue":null,"first_page":"4841","last_page":"4846"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9976999759674072,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10136","display_name":"Statistical Methods and Inference","score":0.9976000189781189,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9973000288009644,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/estimator","display_name":"Estimator","score":0.702602207660675},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.6668896675109863},{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.5544341802597046},{"id":"https://openalex.org/keywords/augmented-lagrangian-method","display_name":"Augmented Lagrangian method","score":0.5184845328330994},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.5146732926368713},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.49904298782348633},{"id":"https://openalex.org/keywords/simultaneous-perturbation-stochastic-approximation","display_name":"Simultaneous perturbation stochastic approximation","score":0.4921073317527771},{"id":"https://openalex.org/keywords/stochastic-approximation","display_name":"Stochastic approximation","score":0.48142382502555847},{"id":"https://openalex.org/keywords/lagrangian-relaxation","display_name":"Lagrangian relaxation","score":0.4686645269393921},{"id":"https://openalex.org/keywords/optimization-problem","display_name":"Optimization problem","score":0.46240168809890747},{"id":"https://openalex.org/keywords/iterated-function","display_name":"Iterated function","score":0.4542350172996521},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.3948459029197693},{"id":"https://openalex.org/keywords/stochastic-process","display_name":"Stochastic process","score":0.38796567916870117},{"id":"https://openalex.org/keywords/applied-mathematics","display_name":"Applied mathematics","score":0.3766833543777466},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.3357531428337097}],"concepts":[{"id":"https://openalex.org/C185429906","wikidata":"https://www.wikidata.org/wiki/Q1130160","display_name":"Estimator","level":2,"score":0.702602207660675},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.6668896675109863},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.5544341802597046},{"id":"https://openalex.org/C150452318","wikidata":"https://www.wikidata.org/wiki/Q4820432","display_name":"Augmented Lagrangian method","level":2,"score":0.5184845328330994},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.5146732926368713},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.49904298782348633},{"id":"https://openalex.org/C2779880469","wikidata":"https://www.wikidata.org/wiki/Q17084424","display_name":"Simultaneous perturbation stochastic approximation","level":3,"score":0.4921073317527771},{"id":"https://openalex.org/C55479107","wikidata":"https://www.wikidata.org/wiki/Q97663916","display_name":"Stochastic approximation","level":3,"score":0.48142382502555847},{"id":"https://openalex.org/C91765299","wikidata":"https://www.wikidata.org/wiki/Q3424292","display_name":"Lagrangian relaxation","level":2,"score":0.4686645269393921},{"id":"https://openalex.org/C137836250","wikidata":"https://www.wikidata.org/wiki/Q984063","display_name":"Optimization problem","level":2,"score":0.46240168809890747},{"id":"https://openalex.org/C140479938","wikidata":"https://www.wikidata.org/wiki/Q5254619","display_name":"Iterated function","level":2,"score":0.4542350172996521},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.3948459029197693},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.38796567916870117},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3766833543777466},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3357531428337097},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cdc.2003.1272362","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc.2003.1272362","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"42nd IEEE International Conference on Decision and Control (IEEE Cat. No.03CH37475)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.75}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":12,"referenced_works":["https://openalex.org/W1518931405","https://openalex.org/W2024767524","https://openalex.org/W2114757210","https://openalex.org/W2118943752","https://openalex.org/W2119792915","https://openalex.org/W2133626316","https://openalex.org/W2161142726","https://openalex.org/W2334782222","https://openalex.org/W2531891978","https://openalex.org/W2798766386","https://openalex.org/W4244633107","https://openalex.org/W4302033506"],"related_works":["https://openalex.org/W2360271985","https://openalex.org/W2131718214","https://openalex.org/W2002987466","https://openalex.org/W2070276483","https://openalex.org/W2038213655","https://openalex.org/W2079907724","https://openalex.org/W2546134129","https://openalex.org/W1972437117","https://openalex.org/W2467306310","https://openalex.org/W1973889396"],"abstract_inverted_index":{"Consider":[0],"the":[1,14,17,31,34,42,46,53,56,64,68,71,78,85,88,113,127,134,146,167,174,178,190,193,210,214,218,240,250,252,254,256,260,264,268,302,309],"problem":[2,18,26],"of":[3,13,45,55,67,94,112,121,126,177,192,213,270],"a":[4,11,23,91,95,108,122,244,282,294],"constrained":[5,219],"Markov":[6,48],"Decision":[7],"Process":[8],"(MDP).":[9],"Under":[10],"parameterization":[12],"control":[15,57],"strategies,":[16],"can":[19,116,160,196],"be":[20,117,161,197,203,288],"transformed":[21],"into":[22],"non-linear":[24,28],"optimization":[25,89,220],"with":[27,98,173,184,237],"constraints.":[29],"Both":[30],"cost":[32,69],"and":[33,70,222,248,272],"constraints":[35],"are":[36,50,59,143],"stationary":[37,79],"averages.":[38],"We":[39,208,298],"assume":[40,155],"that":[41,115,156,189,279],"transition":[43],"probabilities":[44],"underlying":[47],"chain":[49],"unknown:":[51],"only":[52,292],"values":[54,66],"variables":[58],"known,":[60],"as":[61,63,133,145,164,201],"well":[62],"instantaneous":[65],"constraints,":[72],"so":[73,150,163,188,224],"no":[74],"analytical":[75],"expression":[76],"for":[77,205,217,267],"averages":[80],"is":[81,102,152,234],"available.":[82],"To":[83],"find":[84],"solution":[86],"to":[87,154,165,230,305],"problem,":[90],"stochastic":[92,194,215],"version":[93,125],"primal/dual":[96],"method":[97,131],"an":[99],"augmented":[100],"Lagrangian":[101],"used.":[103],"The":[104],"updating":[105],"scheme":[106],"uses":[107],"\"measure":[109],"valued\"":[110],"estimator":[111],"gradients":[114],"interpreted":[118],"in":[119,181,308],"terms":[120],"finite":[123,139,182],"horizon":[124,140,183],"Perturbation":[128],"Analysis":[129],"(PA)":[130],"known":[132],"\"perturbation":[135],"realization":[136],"factors\".":[137],"Most":[138],"derivative":[141],"estimators":[142,180],"consistent":[144],"sample":[147,186],"size":[148],"grows,":[149],"it":[151],"common":[153,236],"large":[157],"enough":[158],"samples":[159],"observed":[162],"make":[166],"bias":[168,212,241,271],"negligible.":[169],"This":[170],"paper":[171],"deals":[172],"actual":[175],"implementations":[176],"gradient":[179],"small":[185],"sizes,":[187],"iterates":[191],"approximation":[195,216],"performed":[198],"very":[199,235],"often,":[200],"would":[202],"required":[204],"on-line":[206],"learning.":[207],"identify":[209],"asymptotic":[211],"method,":[221],"by":[223],"doing":[225],"we":[226,262],"propose":[227],"several":[228],"means":[229],"correct":[231],"it.":[232],"As":[233],"these":[238],"problems,":[239],"correction":[242],"introduces":[243],"conflict":[245],"between":[246],"precision":[247],"speed:":[249],"smaller":[251],"bias,":[253],"slower":[255,295],"reaction":[257],"time.":[258],"In":[259],"sequel,":[261],"present":[263],"theoretical":[265],"basis":[266],"study":[269],"learning":[273],"rate.":[274],"Our":[275],"experimental":[276],"results":[277,300],"indicate":[278],"smoothing":[280],"at":[281,290,293],"faster":[283],"time":[284,296],"scale":[285],"may":[286],"not":[287],"necessary":[289],"all,":[291],"scale.":[297],"include":[299],"where":[301],"algorithms":[303],"have":[304],"track":[306],"changes":[307],"environment.":[310]},"counts_by_year":[{"year":2021,"cited_by_count":3},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2012,"cited_by_count":3}],"updated_date":"2026-06-11T09:08:48.828518","created_date":"2025-10-10T00:00:00"}