{"id":"https://openalex.org/W2123157758","doi":"https://doi.org/10.1145/1143844.1143963","title":"Probabilistic inference for solving discrete and continuous state Markov Decision Processes","display_name":"Probabilistic inference for solving discrete and continuous state Markov Decision Processes","publication_year":2006,"publication_date":"2006-01-01","ids":{"openalex":"https://openalex.org/W2123157758","doi":"https://doi.org/10.1145/1143844.1143963","mag":"2123157758"},"language":"en","primary_location":{"id":"doi:10.1145/1143844.1143963","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1143844.1143963","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd international conference on Machine learning  - ICML '06","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065672819","display_name":"Marc Toussaint","orcid":"https://orcid.org/0000-0002-5487-6767"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":true,"raw_author_name":"Marc Toussaint","raw_affiliation_strings":["University of Edinburgh, Edinburgh, UK","University of Edinburgh, Edinburgh , UK"],"affiliations":[{"raw_affiliation_string":"University of Edinburgh, Edinburgh, UK","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"University of Edinburgh, Edinburgh , UK","institution_ids":["https://openalex.org/I98677209"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5007901825","display_name":"Amos Storkey","orcid":"https://orcid.org/0000-0002-8100-506X"},"institutions":[{"id":"https://openalex.org/I98677209","display_name":"University of Edinburgh","ror":"https://ror.org/01nrxwf90","country_code":"GB","type":"education","lineage":["https://openalex.org/I98677209"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Amos Storkey","raw_affiliation_strings":["University of Edinburgh, Edinburgh, UK","University of Edinburgh, Edinburgh , UK"],"affiliations":[{"raw_affiliation_string":"University of Edinburgh, Edinburgh, UK","institution_ids":["https://openalex.org/I98677209"]},{"raw_affiliation_string":"University of Edinburgh, Edinburgh , UK","institution_ids":["https://openalex.org/I98677209"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5065672819"],"corresponding_institution_ids":["https://openalex.org/I98677209"],"apc_list":null,"apc_paid":null,"fwci":30.7037,"has_fulltext":false,"cited_by_count":501,"citation_normalized_percentile":{"value":0.99749292,"is_in_top_1_percent":true,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":96,"max":100},"biblio":{"volume":null,"issue":null,"first_page":"945","last_page":"952"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11303","display_name":"Bayesian Modeling and Causal Inference","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12814","display_name":"Gaussian Processes and Bayesian Inference","score":0.9961000084877014,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9898999929428101,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/inference","display_name":"Inference","score":0.7954072952270508},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6542586088180542},{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.6116649508476257},{"id":"https://openalex.org/keywords/hidden-markov-model","display_name":"Hidden Markov model","score":0.5399349927902222},{"id":"https://openalex.org/keywords/approximate-inference","display_name":"Approximate inference","score":0.48428457975387573},{"id":"https://openalex.org/keywords/probabilistic-logic","display_name":"Probabilistic logic","score":0.46491390466690063},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4407287538051605},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.4260701835155487},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4017077386379242},{"id":"https://openalex.org/keywords/markov-process","display_name":"Markov process","score":0.38641858100891113},{"id":"https://openalex.org/keywords/algorithm","display_name":"Algorithm","score":0.3329853415489197},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.24970322847366333}],"concepts":[{"id":"https://openalex.org/C2776214188","wikidata":"https://www.wikidata.org/wiki/Q408386","display_name":"Inference","level":2,"score":0.7954072952270508},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6542586088180542},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.6116649508476257},{"id":"https://openalex.org/C23224414","wikidata":"https://www.wikidata.org/wiki/Q176769","display_name":"Hidden Markov model","level":2,"score":0.5399349927902222},{"id":"https://openalex.org/C2777472644","wikidata":"https://www.wikidata.org/wiki/Q16968992","display_name":"Approximate inference","level":3,"score":0.48428457975387573},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.46491390466690063},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4407287538051605},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4260701835155487},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4017077386379242},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.38641858100891113},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3329853415489197},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.24970322847366333},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"doi:10.1145/1143844.1143963","is_oa":false,"landing_page_url":"https://doi.org/10.1145/1143844.1143963","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the 23rd international conference on Machine learning  - ICML '06","raw_type":"proceedings-article"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.60.9444","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.60.9444","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www.marc-toussaint.net/publications/toussaint-storkey-06-icml.ps.gz","raw_type":"text"},{"id":"pmh:oai:CiteSeerX.psu:10.1.1.70.7232","is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.70.7232","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://ml.cs.tu-berlin.de/~mtoussai/publications/toussaint-storkey-06-icml.pdf","raw_type":"text"}],"best_oa_location":null,"sustainable_development_goals":[{"score":0.800000011920929,"id":"https://metadata.un.org/sdg/16","display_name":"Peace, Justice and strong institutions"}],"awards":[],"funders":[{"id":"https://openalex.org/F4320320879","display_name":"Deutsche Forschungsgemeinschaft","ror":"https://ror.org/018mejw64"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":14,"referenced_works":["https://openalex.org/W51537834","https://openalex.org/W1515272691","https://openalex.org/W1528056001","https://openalex.org/W1588316674","https://openalex.org/W1631187438","https://openalex.org/W1650504995","https://openalex.org/W2097668489","https://openalex.org/W2110575115","https://openalex.org/W2162401674","https://openalex.org/W2408670836","https://openalex.org/W2911283634","https://openalex.org/W2914656440","https://openalex.org/W3103882218","https://openalex.org/W3104119384"],"related_works":["https://openalex.org/W187740018","https://openalex.org/W2162286586","https://openalex.org/W4255368532","https://openalex.org/W4226287370","https://openalex.org/W2114556850","https://openalex.org/W2159992248","https://openalex.org/W2215785064","https://openalex.org/W1625332390","https://openalex.org/W2752328056","https://openalex.org/W2951574453"],"abstract_inverted_index":{"Inference":[0],"in":[1,42,102,110,128],"Markov":[2],"Decision":[3],"Processes":[4],"has":[5],"recently":[6],"received":[7],"interest":[8],"as":[9,23],"a":[10,24,120],"means":[11],"to":[12,26],"infer":[13],"goals":[14],"of":[15,33],"an":[16,62,92],"observed":[17],"action,":[18],"policy":[19],"recognition,":[20],"and":[21,89,123],"also":[22],"tool":[25],"compute":[27],"policies.":[28,69],"A":[29],"particularly":[30],"interesting":[31],"aspect":[32],"the":[34,80,111],"approach":[35],"is":[36,100],"that":[37,76,103],"any":[38,104],"existing":[39],"inference":[40,105,118],"technique":[41,106],"DBNs":[43],"now":[44],"becomes":[45],"available":[46],"for":[47,66,85,116],"answering":[48],"behavioral":[49],"question--including":[50],"those":[51],"on":[52,119],"continuous,":[53],"factorial,":[54],"or":[55],"hierarchical":[56],"state":[57,126],"representations.":[58],"Here":[59],"we":[60,73],"present":[61],"Expectation":[63],"Maximization":[64],"algorithm":[65,99],"computing":[67],"optimal":[68,131],"Unlike":[70],"previous":[71],"approaches":[72],"can":[74,107],"show":[75],"this":[77,115],"actually":[78],"optimizes":[79],"discounted":[81],"expected":[82],"future":[83],"return":[84],"arbitrary":[86],"reward":[87],"functions":[88],"without":[90],"assuming":[91],"ad":[93],"hoc":[94],"finite":[95],"total":[96],"time.":[97],"The":[98],"generic":[101],"be":[108],"utilized":[109],"E-step.":[112],"We":[113],"demonstrate":[114],"exact":[117],"discrete":[121],"maze":[122],"Gaussian":[124],"belief":[125],"propagation":[127],"continuous":[129],"stochastic":[130],"control":[132],"problems.":[133]},"counts_by_year":[{"year":2026,"cited_by_count":1},{"year":2025,"cited_by_count":10},{"year":2024,"cited_by_count":7},{"year":2023,"cited_by_count":9},{"year":2022,"cited_by_count":9},{"year":2021,"cited_by_count":34},{"year":2020,"cited_by_count":26},{"year":2019,"cited_by_count":27},{"year":2018,"cited_by_count":14},{"year":2017,"cited_by_count":12},{"year":2016,"cited_by_count":10},{"year":2015,"cited_by_count":44},{"year":2014,"cited_by_count":39},{"year":2013,"cited_by_count":57},{"year":2012,"cited_by_count":44}],"updated_date":"2026-04-04T16:13:02.066488","created_date":"2025-10-10T00:00:00"}