{"id":"https://openalex.org/W1504265534","doi":"https://doi.org/10.1109/cdc.2002.1184811","title":"Self learning control of constrained Markov chains - a gradient approach","display_name":"Self learning control of constrained Markov chains - a gradient approach","publication_year":2003,"publication_date":"2003-08-27","ids":{"openalex":"https://openalex.org/W1504265534","doi":"https://doi.org/10.1109/cdc.2002.1184811","mag":"1504265534"},"language":"en","primary_location":{"id":"doi:10.1109/cdc.2002.1184811","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc.2002.1184811","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 41st IEEE Conference on Decision and Control, 2002.","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5043584689","display_name":"Felisa V\u00e1zquez Abad","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":true,"raw_author_name":"F.V. Abad","raw_affiliation_strings":["D\u00e9partment d Informatique et Recherche op\u00e9rationelle, Universit\u00e9 de Montreal, QUE, Canada"],"affiliations":[{"raw_affiliation_string":"D\u00e9partment d Informatique et Recherche op\u00e9rationelle, Universit\u00e9 de Montreal, QUE, Canada","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5068804090","display_name":"Vikram Krishnamurthy","orcid":"https://orcid.org/0000-0002-4170-6056"},"institutions":[{"id":"https://openalex.org/I165779595","display_name":"University of Melbourne","ror":"https://ror.org/01ej9dk98","country_code":"AU","type":"education","lineage":["https://openalex.org/I165779595"]}],"countries":["AU"],"is_corresponding":false,"raw_author_name":"V. Krishnamurthy","raw_affiliation_strings":["Department of Electrical and Electronic Engineering, University of Melbourne, VIC, Australia"],"affiliations":[{"raw_affiliation_string":"Department of Electrical and Electronic Engineering, University of Melbourne, VIC, Australia","institution_ids":["https://openalex.org/I165779595"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032264786","display_name":"Kipp Martin","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"K. Martin","raw_affiliation_strings":["D\u00e9partment d Informatique et Recherche op\u00e9rationelle, Universit\u00e9 de Montreal, QUE, Canada"],"affiliations":[{"raw_affiliation_string":"D\u00e9partment d Informatique et Recherche op\u00e9rationelle, Universit\u00e9 de Montreal, QUE, Canada","institution_ids":["https://openalex.org/I70931966"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5063875266","display_name":"Irina Baltcheva","orcid":null},"institutions":[{"id":"https://openalex.org/I70931966","display_name":"Universit\u00e9 de Montr\u00e9al","ror":"https://ror.org/0161xgx34","country_code":"CA","type":"education","lineage":["https://openalex.org/I70931966"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"I. Baltcheva","raw_affiliation_strings":["D\u00e9partment d Informatique et Recherche op\u00e9rationelle, Universit\u00e9 de Montreal, QUE, Canada"],"affiliations":[{"raw_affiliation_string":"D\u00e9partment d Informatique et Recherche op\u00e9rationelle, Universit\u00e9 de Montreal, QUE, Canada","institution_ids":["https://openalex.org/I70931966"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5043584689"],"corresponding_institution_ids":["https://openalex.org/I70931966"],"apc_list":null,"apc_paid":null,"fwci":0.3404,"has_fulltext":false,"cited_by_count":9,"citation_normalized_percentile":{"value":0.57069892,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":96},"biblio":{"volume":"2","issue":null,"first_page":"1940","last_page":"1945"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12056","display_name":"Markov Chains and Monte Carlo Methods","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12056","display_name":"Markov Chains and Monte Carlo Methods","score":0.9944999814033508,"subfield":{"id":"https://openalex.org/subfields/2613","display_name":"Statistics and Probability"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10320","display_name":"Neural Networks and Applications","score":0.9864000082015991,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10791","display_name":"Advanced Control Systems Optimization","score":0.9767000079154968,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-chain","display_name":"Markov chain","score":0.710873007774353},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.5860608816146851},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.47980359196662903},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.45911118388175964},{"id":"https://openalex.org/keywords/examples-of-markov-chains","display_name":"Examples of Markov chains","score":0.41875913739204407},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.3342271149158478},{"id":"https://openalex.org/keywords/balance-equation","display_name":"Balance equation","score":0.3252214193344116},{"id":"https://openalex.org/keywords/markov-model","display_name":"Markov model","score":0.2941353917121887},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.20158717036247253},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.19962522387504578},{"id":"https://openalex.org/keywords/statistics","display_name":"Statistics","score":0.07946240901947021}],"concepts":[{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.710873007774353},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5860608816146851},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.47980359196662903},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.45911118388175964},{"id":"https://openalex.org/C76408418","wikidata":"https://www.wikidata.org/wiki/Q5419314","display_name":"Examples of Markov chains","level":5,"score":0.41875913739204407},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3342271149158478},{"id":"https://openalex.org/C158917313","wikidata":"https://www.wikidata.org/wiki/Q4849964","display_name":"Balance equation","level":4,"score":0.3252214193344116},{"id":"https://openalex.org/C163836022","wikidata":"https://www.wikidata.org/wiki/Q6771326","display_name":"Markov model","level":3,"score":0.2941353917121887},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.20158717036247253},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.19962522387504578},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.07946240901947021}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/cdc.2002.1184811","is_oa":false,"landing_page_url":"https://doi.org/10.1109/cdc.2002.1184811","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 41st IEEE Conference on Decision and Control, 2002.","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.7200000286102295,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":9,"referenced_works":["https://openalex.org/W34047206","https://openalex.org/W1518931405","https://openalex.org/W1576452626","https://openalex.org/W2024767524","https://openalex.org/W2089256099","https://openalex.org/W2161142726","https://openalex.org/W2334782222","https://openalex.org/W2798766386","https://openalex.org/W4254313632"],"related_works":["https://openalex.org/W4235599811","https://openalex.org/W3099350264","https://openalex.org/W2392270973","https://openalex.org/W292716529","https://openalex.org/W4232946018","https://openalex.org/W1892287825","https://openalex.org/W2149026075","https://openalex.org/W2326234159","https://openalex.org/W2293264007","https://openalex.org/W2379651310"],"abstract_inverted_index":{"We":[0],"present":[1],"stochastic":[2,22,109,119],"approximation":[3,23],"algorithms":[4,24,60,68,96],"for":[5],"computing":[6],"the":[7,28,31,37,41,67,71,83,95],"locally":[8],"optimal":[9],"policy":[10],"of":[11,27,30,82,115],"a":[12],"constrained":[13,108],"average":[14],"cost":[15,32],"finite":[16],"state":[17],"Markov":[18],"decision":[19],"process.":[20],"The":[21,105],"require":[25,79],"computation":[26],"gradient":[29,50,110],"function":[33],"with":[34],"respect":[35],"to":[36,57],"parameter":[38],"that":[39],"characterizes":[40],"randomized":[42],"policy.":[43],"This":[44],"is":[45,113],"computed":[46],"by":[47],"simulation":[48,74],"based":[49,75,107],"estimation":[51],"schemes":[52],"involving":[53],"weak":[54],"derivatives.":[55],"Similar":[56],"neuro-dynamic":[58,92],"programming":[59,93],"(e.g.":[61],"Q-learning":[62],"or":[63],"temporal":[64],"difference":[65],"methods),":[66],"proposed":[69,97,112],"in":[70,118],"paper":[72],"are":[73],"and":[76,101],"do":[77],"not":[78],"explicit":[80],"knowledge":[81],"underlying":[84],"parameters":[85],"such":[86],"as":[87],"transition":[88],"probabilities.":[89],"However,":[90],"unlike":[91],"methods,":[94],"can":[98],"handle":[99],"constraints":[100],"time":[102],"varying":[103],"parameters.":[104],"multiplier":[106],"algorithm":[111],"also":[114],"independent":[116],"interest":[117],"approximation.":[120]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2016,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}