{"id":"https://openalex.org/W2182995676","doi":null,"title":"Approximate solution methods for partially observable markov and semi-markov decision processes","display_name":"Approximate solution methods for partially observable markov and semi-markov decision processes","publication_year":2006,"publication_date":"2006-01-01","ids":{"openalex":"https://openalex.org/W2182995676","mag":"2182995676"},"language":"en","primary_location":{"id":"mag:2182995676","doi":null,"is_oa":false,"landing_page_url":"http://dl.acm.org/citation.cfm?id=1195372","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5039911471","display_name":"Dimitri P. Bertsekas","orcid":"https://orcid.org/0000-0001-6909-7208","observed_orcids":["https://orcid.org/0000-0001-6909-7208"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Dimitri P. Bertsekas","raw_affiliation_strings":["Massachusetts Institute Of Technology#TAB#"],"raw_orcid":"https://orcid.org/0000-0001-6909-7208","affiliations":[{"raw_affiliation_string":"Massachusetts Institute Of Technology#TAB#","institution_ids":["https://openalex.org/I63966007"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5014937312","display_name":"Huizhen Yu","orcid":"https://orcid.org/0000-0002-3673-0094","observed_orcids":["https://orcid.org/0000-0002-3673-0094"]},"institutions":[{"id":"https://openalex.org/I63966007","display_name":"Massachusetts Institute of Technology","ror":"https://ror.org/042nb2s44","country_code":"US","type":"education","lineage":["https://openalex.org/I63966007"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huizhen Yu","raw_affiliation_strings":["Massachusetts Institute Of Technology#TAB#"],"raw_orcid":"https://orcid.org/0000-0002-3673-0094","affiliations":[{"raw_affiliation_string":"Massachusetts Institute Of Technology#TAB#","institution_ids":["https://openalex.org/I63966007"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I63966007"],"apc_list":null,"apc_paid":null,"fwci":2.117,"has_fulltext":false,"cited_by_count":30,"citation_normalized_percentile":{"value":0.86930838,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":87,"max":97},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9487000107765198,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9487000107765198,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.027499999850988388,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11195","display_name":"Simulation Techniques and Applications","score":0.0012000000569969416,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/partially-observable-markov-decision-processes","display_name":"partially observable Markov decision processes","score":0.9940000176429749},{"id":"https://openalex.org/keywords/semi-markov-decision-process","display_name":"semi-Markov decision process","score":0.9350000023841858},{"id":"https://openalex.org/keywords/average-cost-criterion","display_name":"average cost criterion","score":0.9319999814033508},{"id":"https://openalex.org/keywords/optimal-stationary-policy","display_name":"stationary optimal policy","score":0.5910000205039978},{"id":"https://openalex.org/keywords/linear-programming-bounds","display_name":"linear programming bounds","score":0.578000009059906},{"id":"https://openalex.org/keywords/finite-state-controllers","display_name":"finite-state controllers","score":0.8539999723434448},{"id":"https://openalex.org/keywords/convergence-rate-analysis","display_name":"convergence rate analysis","score":0.5180000066757202}],"concepts":[{"id":"https://openalex.org/C17098449","wikidata":"https://www.wikidata.org/wiki/Q176814","display_name":"Partially observable Markov decision process","level":4,"score":0.9269664287567139},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.7268567085266113},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.688997745513916},{"id":"https://openalex.org/C32848918","wikidata":"https://www.wikidata.org/wiki/Q845789","display_name":"Observable","level":2,"score":0.5904669761657715},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5347522497177124},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5336739420890808},{"id":"https://openalex.org/C98763669","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov chain","level":2,"score":0.5131168961524963},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.4889920949935913},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.4768608510494232},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3939356803894043},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.057311028242111206},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2182995676","doi":null,"is_oa":false,"landing_page_url":"http://dl.acm.org/citation.cfm?id=1195372","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":56,"referenced_works":["https://openalex.org/W75971611","https://openalex.org/W1483935885","https://openalex.org/W1491973539","https://openalex.org/W1507222174","https://openalex.org/W1518539242","https://openalex.org/W1518931405","https://openalex.org/W1524146404","https://openalex.org/W1529558080","https://openalex.org/W1532022778","https://openalex.org/W1539216098","https://openalex.org/W1541084404","https://openalex.org/W1553175366","https://openalex.org/W1576452626","https://openalex.org/W1588937771","https://openalex.org/W1597303641","https://openalex.org/W1640774615","https://openalex.org/W1733520681","https://openalex.org/W1967037758","https://openalex.org/W2011418219","https://openalex.org/W2018444179","https://openalex.org/W2021649043","https://openalex.org/W2029631728","https://openalex.org/W2040434095","https://openalex.org/W2044375425","https://openalex.org/W2046637502","https://openalex.org/W2050941272","https://openalex.org/W2071370395","https://openalex.org/W2072931156","https://openalex.org/W2077957803","https://openalex.org/W2089211960","https://openalex.org/W2095267417","https://openalex.org/W2096132608","https://openalex.org/W2097616825","https://openalex.org/W2098432798","https://openalex.org/W2099111195","https://openalex.org/W2100677568","https://openalex.org/W2107338824","https://openalex.org/W2108682071","https://openalex.org/W2109910161","https://openalex.org/W2115502112","https://openalex.org/W2119567691","https://openalex.org/W2120465407","https://openalex.org/W2121863487","https://openalex.org/W2139418546","https://openalex.org/W2144446635","https://openalex.org/W2148024708","https://openalex.org/W2149476049","https://openalex.org/W2155027007","https://openalex.org/W2156737235","https://openalex.org/W2159022486","https://openalex.org/W2160067530","https://openalex.org/W2258043381","https://openalex.org/W2415292238","https://openalex.org/W2802633207","https://openalex.org/W3140571750","https://openalex.org/W3150304496"],"related_works":["https://openalex.org/W1588937771","https://openalex.org/W4298248375","https://openalex.org/W2149476049","https://openalex.org/W180325379","https://openalex.org/W2998102890","https://openalex.org/W2169294731","https://openalex.org/W2766564508","https://openalex.org/W2065087844","https://openalex.org/W1979259161","https://openalex.org/W3196739113"],"abstract_inverted_index":{"We":[0,82,215,259,329,347,403,439],"consider":[1,321],"approximation":[2,29,71],"methods":[3,324,368],"for":[4,31,78,87,169,287,325,369,373,387,400,416,457,496],"discrete-time":[5],"infinite-horizon":[6],"partially":[7,44],"observable":[8,45],"Markov":[9],"and":[10,15,39,47,183,192,244,274,327,371,383,397,429,443,450],"semi-Markov":[11,43,272],"decision":[12],"processes":[13],"(POMDP":[14],"POSMDP).":[16],"One":[17],"of":[18,22,67,90,108,118,137,162,177,186,219,225,264,291,306,411,426,434,437,446,474,477,483],"the":[19,35,57,105,133,138,145,160,163,175,180,184,212,226,245,261,265,281,292,307,353,362,365,408,412,424,427,458,472,475,481,497],"main":[20],"contributions":[21],"this":[23,405,468],"thesis":[24],"is":[25,64,115,199],"a":[26,84,112,116,170,205,339,344,381,388,453,484,493],"lower":[27,69,130,196,223,266,289,304],"cost":[28,37,60,70,135,148,173,230,247,276,294,299,310,377,432,448,462],"method":[30,63,283],"finite-space":[32],"POMDPs":[33,124,143,326,370,417],"with":[34,56,94,144,418],"average":[36,147,229,298,376,431,461,488],"criterion,":[38],"its":[40],"extensions":[41],"to":[42,54,104,132,159,179,253,352,463],"problems":[46,55,239,273],"constrained":[48,297,308],"POMDP":[49,80,140],"problems,":[50,277],"as":[51,53,167,343],"well":[52],"undiscounted":[58],"total":[59,275],"criterion.":[61],"\r\nOur":[62],"an":[65,331,350],"extension":[66],"several":[68],"schemes,":[72],"proposed":[73,399],"individually":[74],"by":[75,236,315],"various":[76,435,441],"authors,":[77],"discounted":[79,401],"problems.":[81,402],"introduce":[83],"unified":[85],"framework":[86],"viewing":[88],"all":[89],"these":[91,220,447],"schemes":[92],"together":[93],"some":[95],"new":[96,454],"ones.":[97],"In":[98],"particular,":[99],"we":[100,278,301,379,422,451,470],"establish":[101],"that":[102,128,189,217,280,303,337],"due":[103],"special":[106],"structure":[107,161],"hidden":[109],"states":[110],"in":[111,203,210,407],"POMDP,":[113],"there":[114],"class":[117,476],"approximating":[119,246],"processes,":[120],"which":[121],"are":[122,150,190],"either":[123],"or":[125],"belief":[126],"MDPs,":[127,378],"provide":[129,349,380],"bounds":[131,224,267,290,305],"optimal":[134,164,172,187,213,227,293,309,459,486],"function":[136,231,311],"original":[139],"problem.":[141],"\r\nTheoretically,":[142],"long-run":[146],"criterion":[149],"still":[151],"not":[152,201],"fully":[153],"understood.":[154],"The":[155],"major":[156],"difficulties":[157],"relate":[158],"solutions,":[165],"such":[166],"conditions":[168],"constant":[171,485],"function,":[174],"existence":[176,185],"solutions":[178],"optimality":[181],"equations,":[182],"policies":[188],"stationary":[191,256],"deterministic.":[193],"Thus,":[194],"our":[195],"bound":[197],"result":[198,491],"useful":[200],"only":[202],"providing":[204],"computational":[206],"method,":[207],"but":[208],"also":[209,251,320,360],"characterizing":[211],"solution.":[214],"show":[216,279,302,440],"regardless":[218],"theoretical":[221,494],"difficulties,":[222],"liminf":[228,430,460,487],"can":[232,249,284,312],"be":[233,250,285,313,464],"computed":[234,314],"efficiently":[235],"solving":[237,316],"modified":[238],"using":[240],"multichain":[241],"MDP":[242],"algorithms,":[243],"functions":[248,433],"used":[252],"obtain":[254],"suboptimal":[255],"control":[257],"policies.":[258,438],"prove":[260,471],"asymptotic":[262],"convergence":[263,382,384],"under":[268,480],"certain":[269],"assumptions.":[270],"\r\nFor":[271],"same":[282],"applied":[286],"computing":[288],"function.":[295],"For":[296,375],"POMDPs,":[300],"finite-dimensional":[317],"LPs.":[318],"\r\nWe":[319],"reinforcement":[322,366],"learning":[323,367],"MDPs.":[328,374],"propose":[330],"actor-critic":[332],"type":[333],"policy":[334,341,413],"gradient":[335,414],"algorithm":[336,356,406,415],"uses":[338],"structured":[340],"known":[342],"finite-state":[345,419,478,498],"controller.":[346],"thus":[348],"alternative":[351],"earlier":[354],"actor-only":[355],"GPOMDP.":[357],"Our":[358],"work":[359],"clarifies":[361],"relationship":[363],"between":[364],"those":[372],"rate":[385],"analysis":[386],"least":[389],"squares":[390],"temporal":[391],"difference":[392],"(TD)":[393],"algorithm,":[394],"called":[395],"LSPE,":[396],"previously":[398],"use":[404],"critic":[409],"portion":[410],"controllers.":[420],"\r\nFinally,":[421],"investigate":[423],"properties":[425,445],"limsup":[428],"types":[436],"convexity":[442],"concavity":[444],"functions,":[449],"give":[452],"necessary":[455],"condition":[456],"constant.":[465],"Based":[466],"on":[467],"condition,":[469],"near-optimality":[473],"controllers":[479],"assumption":[482],"cost.":[489],"This":[490],"provides":[492],"guarantee":[495],"controller":[499],"approach.":[500],"(Copies":[501],"available":[502],"exclusively":[503],"from":[504],"MIT":[505],"Libraries,":[506],"Rm.":[507],"14-0551,":[508],"Cambridge,":[509],"MA":[510],"02139-4307.":[511],"Ph.":[512],"617-253-5668;":[513],"Fax":[514],"617-253-1690.)":[515]},"counts_by_year":[{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2018,"cited_by_count":3},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":2},{"year":2014,"cited_by_count":3},{"year":2013,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}