{"id":"https://openalex.org/W32198084","doi":null,"title":"Learning to Solve Markovian Decision Processes","display_name":"Learning to Solve Markovian Decision Processes","publication_year":1993,"publication_date":"1993-11-01","ids":{"openalex":"https://openalex.org/W32198084","mag":"32198084"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.42.6698","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.42.6698","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www-anw.cs.umass.edu/cgi-bin/getfile/pub/anw/pub/singh/Thesis/MDPlearn.ps.Z","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5065366930","display_name":"Satinder Pal Singh","orcid":"https://orcid.org/0000-0002-2736-7641","observed_orcids":["https://orcid.org/0000-0002-2736-7641","https://orcid.org/0000-0002-8215-8295","https://orcid.org/0000-0002-5169-9486","https://orcid.org/0000-0001-9360-7060"]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Satinder Singh","raw_affiliation_strings":["University of Massachusetts: Amherst#TAB#"],"raw_orcid":"https://orcid.org/0000-0002-2736-7641","affiliations":[{"raw_affiliation_string":"University of Massachusetts: Amherst#TAB#","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":["https://openalex.org/A5065366930"],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":4.7164,"has_fulltext":false,"cited_by_count":63,"citation_normalized_percentile":{"value":0.94586741,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":88,"max":97},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/markov-decision-process","display_name":"Markov decision process","score":0.7929999828338623},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.9190000295639038},{"id":"https://openalex.org/keywords/function-approximation","display_name":"function approximation","score":0.6859999895095825},{"id":"https://openalex.org/keywords/evaluation-function-learning","display_name":"evaluation function learning","score":0.47099998593330383},{"id":"https://openalex.org/keywords/accelerated-learning","display_name":"accelerated learning","score":0.4959999918937683},{"id":"https://openalex.org/keywords/transfer-of-training","display_name":"transfer of training","score":0.652999997138977},{"id":"https://openalex.org/keywords/multi-task-learning","display_name":"multi-task learning","score":0.6179999709129333}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8773679733276367},{"id":"https://openalex.org/C106189395","wikidata":"https://www.wikidata.org/wiki/Q176789","display_name":"Markov decision process","level":3,"score":0.796333909034729},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.718308687210083},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.5262048840522766},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5010333061218262},{"id":"https://openalex.org/C37404715","wikidata":"https://www.wikidata.org/wiki/Q380679","display_name":"Dynamic programming","level":2,"score":0.49548184871673584},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.4632592797279358},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.44212955236434937},{"id":"https://openalex.org/C159886148","wikidata":"https://www.wikidata.org/wiki/Q176645","display_name":"Markov process","level":2,"score":0.4281432628631592},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.4164005517959595},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.410602331161499},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.19825023412704468},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.17893320322036743},{"id":"https://openalex.org/C202444582","wikidata":"https://www.wikidata.org/wiki/Q837863","display_name":"Pure mathematics","level":1,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.42.6698","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.42.6698","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www-anw.cs.umass.edu/cgi-bin/getfile/pub/anw/pub/singh/Thesis/MDPlearn.ps.Z","raw_type":"text"},{"id":"pmh:oai:scholarworks.umass.edu:dissertations-8863","doi":null,"is_oa":false,"landing_page_url":"https://scholarworks.umass.edu/dissertations/AAI9420690","pdf_url":null,"source":{"id":"https://openalex.org/S4306402057","display_name":"Scholarworks (University of Massachusetts Amherst)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I24603500","host_organization_name":"University of Massachusetts Amherst","host_organization_lineage":["https://openalex.org/I24603500"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral Dissertations Available from Proquest","raw_type":"text"},{"id":"mag:32198084","doi":null,"is_oa":false,"landing_page_url":"http://dl.acm.org/citation.cfm?id=192879","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G4361742918","display_name":"Fourth International Conference on Alcohol and Cancer","funder_award_id":"R13AA027725","funder_id":"https://openalex.org/F4320337330","funder_display_name":"National Institute on Alcohol Abuse and Alcoholism"},{"id":"https://openalex.org/G7442776549","display_name":"Metabolism of Carcinogens and Drugs by Human P450s","funder_award_id":"R01GM118122","funder_id":"https://openalex.org/F4320337354","funder_display_name":"National Institute of General Medical Sciences"}],"funders":[{"id":"https://openalex.org/F4320332161","display_name":"National Institutes of Health","ror":"https://ror.org/01cwqze88"},{"id":"https://openalex.org/F4320337330","display_name":"National Institute on Alcohol Abuse and Alcoholism","ror":"https://ror.org/02jzrsm59"},{"id":"https://openalex.org/F4320337354","display_name":"National Institute of General Medical Sciences","ror":"https://ror.org/04q48ey07"}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":4,"referenced_works":["https://openalex.org/W1539452355","https://openalex.org/W1925486849","https://openalex.org/W2148994957","https://openalex.org/W2172159364"],"related_works":["https://openalex.org/W2386526113","https://openalex.org/W3038611842","https://openalex.org/W7100147173","https://openalex.org/W1506832649","https://openalex.org/W6974180586","https://openalex.org/W2552244163","https://openalex.org/W2576525097","https://openalex.org/W7047432649","https://openalex.org/W4398230852","https://openalex.org/W2170141744"],"abstract_inverted_index":{"This":[0,63,89,126,243],"dissertation":[1,90,127,244],"is":[2,64,80,204],"about":[3],"building":[4,313],"learning":[5,36,41,282],"control":[6],"architectures":[7,18,231,263,370],"for":[8,98,143,186,239,309,320,328,344,352],"agents":[9,21],"embedded":[10,20],"in":[11,54,71,181,193,372],"finite,":[12,93],"stationary,":[13],"and":[14,95,105,113,136,257,278,337,360],"Markovian":[15,96],"environments.":[16],"Such":[17],"give":[19],"the":[22,27,51,76,103,146,187,216,226,249,252,258,306,317,325,340,345,349,353,367],"ability":[23],"to":[24,57,84,154,209,235,254,260,267,290,299,315,331,365],"improve":[25,58],"autonomously":[26],"efficiency":[28],"with":[29,82,87],"which":[30],"they":[31],"can":[32,265],"achieve":[33,300],"goals.":[34],"Machine":[35],"researchers":[37],"have":[38,151],"developed":[39,371],"reinforcement":[40],"(RL)":[42],"algorithms":[43,149,203,207],"based":[44],"on":[45,92,170],"dynamic":[46],"programming":[47],"(DP)":[48],"that":[49,75,79,121,138,150,161,211,264,280],"use":[50,106,188],"agent's":[52],"experience":[53],"its":[55,59],"environment":[56,335],"decision":[60,77,341],"policy":[61,78],"incrementally.":[62],"achieved":[65],"by":[66,219,283],"adapting":[67],"all":[68,145,164,224],"evaluation":[69,307,318,326],"function":[70,191,319],"such":[72],"a":[73,108,129,140,159,197,296],"way":[74],"\"greedy\"":[81],"respect":[83],"it":[85,101],"improves":[86],"experience.":[88],"focuses":[91],"stationary":[94],"environments":[97],"two":[99,246],"reasons:":[100],"allows":[102],"development":[104],"of":[107,111,189,200,225,248,286,302,358],"strong":[109],"theory":[110,135],"RL,":[112,256],"there":[114],"are":[115,212,363],"many":[116,240],"challenging":[117],"real-world":[118,241],"RL":[119,137,148,165,182,230,262,274,369],"tasks":[120,311,330,347],"fall":[122],"into":[123],"this":[124,373],"category.":[125],"establishes":[128],"novel":[130],"connection":[131],"between":[132],"stochastic":[133],"approximation":[134],"provides":[139],"uniform":[141],"framework":[142],"understanding":[144],"different":[147,297],"been":[152],"proposed":[153],"date.":[155],"It":[156,271],"also":[157],"highlights":[158],"dimension":[160],"clearly":[162],"separates":[163],"research":[166],"from":[167,288],"prior":[168],"work":[169],"DP.":[171],"Two":[172],"other":[173],"theoretical":[174,227,359],"results":[175,362],"showing":[176],"how":[177],"approximations":[178],"affect":[179],"performance":[180],"provide":[183],"partial":[184],"justification":[185],"compact":[190],"approximators":[192],"RL.":[194],"In":[195],"addition,":[196],"new":[198,368],"family":[199],"\"soft\"":[201],"DP":[202,221],"presented.":[205],"These":[206],"converge":[208],"solutions":[210,217],"more":[213],"robust":[214],"than":[215],"found":[218,343],"classical":[220],"algorithms.":[222],"Despite":[223],"progress,":[228],"conventional":[229],"scale":[232],"poorly":[233],"enough":[234],"make":[236],"them":[237],"impractical":[238],"problems.":[242],"studies":[245],"aspects":[247],"scaling":[250],"issue:":[251],"need":[253,259],"accelerate":[255,281],"build":[261,332],"learn":[266],"solve":[268],"multiple":[269],"tasks.":[270,292,355],"presents":[272],"three":[273],"architectures,":[275],"CQ-L,":[276],"H-DYNA,":[277],"BB-RL,":[279],"facilitating":[284],"transfer":[285,301],"training":[287],"simple":[289,310,329,346],"complex":[291,321,354],"Each":[293],"architecture":[294],"uses":[295,305,324,339],"method":[298],"training;":[303],"CQ-L":[304],"functions":[308,327],"as":[312,348],"blocks":[314],"construct":[316],"tasks,":[322],"H-DYNA":[323],"an":[333],"abstract":[334],"model,":[336],"BB-RL":[338],"policies":[342],"primitive":[350],"actions":[351],"A":[356],"mixture":[357],"empirical":[361],"presented":[364],"support":[366],"dissertation.":[374]},"counts_by_year":[{"year":2021,"cited_by_count":3},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":1},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":2},{"year":2013,"cited_by_count":3},{"year":2012,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}