{"id":"https://openalex.org/W57282082","doi":null,"title":"A causal approach to hierarchical decomposition in reinforcement learning","display_name":"A causal approach to hierarchical decomposition in reinforcement learning","publication_year":2006,"publication_date":"2006-01-01","ids":{"openalex":"https://openalex.org/W57282082","mag":"57282082"},"language":"en","primary_location":{"id":"pmh:oai:CiteSeerX.psu:10.1.1.137.4637","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.137.4637","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www-all.cs.umass.edu/pubs/2005/jonsson_thesis05.pdf","raw_type":"text"},"volume":null,"type":"article","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://scholarworks.umass.edu/dissertations/AAI3212735","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5089808068","display_name":"Andrew G. Barto","orcid":"https://orcid.org/0000-0002-3581-1896","observed_orcids":["https://orcid.org/0000-0002-3581-1896"]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Andrew G. Barto","raw_affiliation_strings":["University of Massachusetts: Amherst#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Massachusetts: Amherst#TAB#","institution_ids":["https://openalex.org/I24603500"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5086943516","display_name":"Anders J\u00f6nsson","orcid":"https://orcid.org/0000-0002-5756-7847","observed_orcids":["https://orcid.org/0000-0002-5756-7847"]},"institutions":[{"id":"https://openalex.org/I24603500","display_name":"University of Massachusetts Amherst","ror":"https://ror.org/0072zz521","country_code":"US","type":"education","lineage":["https://openalex.org/I24603500"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Anders J\u00f6nsson","raw_affiliation_strings":["University of Massachusetts: Amherst#TAB#"],"raw_orcid":"https://orcid.org/0000-0002-5756-7847","affiliations":[{"raw_affiliation_string":"University of Massachusetts: Amherst#TAB#","institution_ids":["https://openalex.org/I24603500"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I24603500"],"apc_list":null,"apc_paid":null,"fwci":0.3536,"has_fulltext":false,"cited_by_count":6,"citation_normalized_percentile":{"value":0.65987817,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":88,"max":93},"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hierarchical-decomposition","display_name":"hierarchical decomposition","score":0.8659999966621399},{"id":"https://openalex.org/keywords/state-abstraction","display_name":"state abstraction","score":0.8500000238418579},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.7699999809265137},{"id":"https://openalex.org/keywords/dynamic-bayesian-networks","display_name":"dynamic Bayesian networks","score":0.871999979019165},{"id":"https://openalex.org/keywords/task-structure","display_name":"task structure","score":0.5139999985694885},{"id":"https://openalex.org/keywords/subtask-decomposition","display_name":"subtask decomposition","score":0.6510000228881836},{"id":"https://openalex.org/keywords/planning","display_name":"planning","score":0.5569999814033508},{"id":"https://openalex.org/keywords/scalable-learning","display_name":"scalable learning","score":0.5450000166893005}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.9190540909767151},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7464545965194702},{"id":"https://openalex.org/C31170391","wikidata":"https://www.wikidata.org/wiki/Q188619","display_name":"Hierarchy","level":2,"score":0.6802196502685547},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.6795287132263184},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6108813285827637},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5971532464027405},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.5543373227119446},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.5201730728149414},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.5145477652549744},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5038835406303406},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.09601792693138123},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C199360897","wikidata":"https://www.wikidata.org/wiki/Q9143","display_name":"Programming language","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C18903297","wikidata":"https://www.wikidata.org/wiki/Q7150","display_name":"Ecology","level":1,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C34447519","wikidata":"https://www.wikidata.org/wiki/Q179522","display_name":"Market economy","level":1,"score":0.0}],"mesh":[],"locations_count":3,"locations":[{"id":"pmh:oai:CiteSeerX.psu:10.1.1.137.4637","doi":null,"is_oa":false,"landing_page_url":"http://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.137.4637","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"http://www-all.cs.umass.edu/pubs/2005/jonsson_thesis05.pdf","raw_type":"text"},{"id":"pmh:oai:scholarworks.umass.edu:dissertations-4292","doi":null,"is_oa":true,"landing_page_url":"https://scholarworks.umass.edu/dissertations/AAI3212735","pdf_url":null,"source":{"id":"https://openalex.org/S4306402240","display_name":"ScholarWorks@UMassAmherst (University of Massachusetts Amherst)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I24603500","host_organization_name":"University of Massachusetts Amherst","host_organization_lineage":["https://openalex.org/I24603500"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral Dissertations Available from Proquest","raw_type":"text"},{"id":"mag:57282082","doi":null,"is_oa":false,"landing_page_url":"https://scholarworks.umass.edu/cgi/viewcontent.cgi?article=4292&context=dissertations","pdf_url":null,"source":null,"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null}],"best_oa_location":{"id":"pmh:oai:scholarworks.umass.edu:dissertations-4292","doi":null,"is_oa":true,"landing_page_url":"https://scholarworks.umass.edu/dissertations/AAI3212735","pdf_url":null,"source":{"id":"https://openalex.org/S4306402240","display_name":"ScholarWorks@UMassAmherst (University of Massachusetts Amherst)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I24603500","host_organization_name":"University of Massachusetts Amherst","host_organization_lineage":["https://openalex.org/I24603500"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Doctoral Dissertations Available from Proquest","raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":69,"referenced_works":["https://openalex.org/W1666476","https://openalex.org/W112199310","https://openalex.org/W123722869","https://openalex.org/W139620917","https://openalex.org/W1486707268","https://openalex.org/W1494114146","https://openalex.org/W1496855202","https://openalex.org/W1517993545","https://openalex.org/W1530444831","https://openalex.org/W1536990779","https://openalex.org/W1543507677","https://openalex.org/W1564393562","https://openalex.org/W1566917201","https://openalex.org/W1583380718","https://openalex.org/W1585546214","https://openalex.org/W1586944634","https://openalex.org/W1598052524","https://openalex.org/W1612195517","https://openalex.org/W1640646391","https://openalex.org/W1650504995","https://openalex.org/W1688218840","https://openalex.org/W1754881896","https://openalex.org/W1896341954","https://openalex.org/W1920714059","https://openalex.org/W1931792391","https://openalex.org/W1966195676","https://openalex.org/W1968768508","https://openalex.org/W1996326480","https://openalex.org/W1997477668","https://openalex.org/W2008906462","https://openalex.org/W2011418219","https://openalex.org/W2020294948","https://openalex.org/W2028145673","https://openalex.org/W2041367235","https://openalex.org/W2042680115","https://openalex.org/W2076337359","https://openalex.org/W2086472796","https://openalex.org/W2089561656","https://openalex.org/W2090170171","https://openalex.org/W2109910161","https://openalex.org/W2110652811","https://openalex.org/W2114451917","https://openalex.org/W2115931363","https://openalex.org/W2117341272","https://openalex.org/W2121517924","https://openalex.org/W2121615264","https://openalex.org/W2121723232","https://openalex.org/W2121863487","https://openalex.org/W2124175081","https://openalex.org/W2124403132","https://openalex.org/W2128281152","https://openalex.org/W2132096648","https://openalex.org/W2143435603","https://openalex.org/W2146855196","https://openalex.org/W2147334703","https://openalex.org/W2153175335","https://openalex.org/W2153947321","https://openalex.org/W2158548602","https://openalex.org/W2159080219","https://openalex.org/W2160808139","https://openalex.org/W2170198047","https://openalex.org/W2334782222","https://openalex.org/W2337392266","https://openalex.org/W2396715201","https://openalex.org/W2548352678","https://openalex.org/W2911546748","https://openalex.org/W2963487185","https://openalex.org/W3011120880","https://openalex.org/W3020831056"],"related_works":["https://openalex.org/W2343320077","https://openalex.org/W1550915716","https://openalex.org/W2411281527","https://openalex.org/W4253921394","https://openalex.org/W2306711933","https://openalex.org/W2121517924","https://openalex.org/W3148270200","https://openalex.org/W4235743572","https://openalex.org/W3000554410","https://openalex.org/W2901329458"],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1,33,57,75,84,108,417],"provides":[2,277,328,357],"a":[3,53,67,95,128,139,157,167,253,257,266,269,329,333,358,379,411],"means":[4,330],"for":[5,17,210,231,285,344],"autonomous":[6],"agents":[7],"to":[8,77,87,105,152,160,180,191,226,244,263,281,300,311,315,331,352,414,420],"improve":[9,398],"their":[10],"action":[11,261],"selection":[12],"strategies":[13],"without":[14,318],"the":[15,50,124,154,194,199,203,224,275,301,307,321,346,391,399],"need":[16],"explicit":[18],"training":[19],"information":[20],"provided":[21],"by":[22,216,394],"an":[23,218,235,278],"informed":[24],"instructor.":[25],"Theoretical":[26],"and":[27,62,149,325],"empirical":[28,373],"results":[29,374,405],"indicate":[30,406],"that":[31,80,99,114,123,172,186,220,297,385,407],"reinforcement":[32,56,74,83,107,162,402,416],"algorithms":[34,58,85,98,396,409],"can":[35,173],"efficiently":[36,312],"determine":[37],"optimal":[38,41],"or":[39],"approximately":[40],"policies":[42],"in":[43,73,111,234,342,378,424],"tasks":[44,113,343,384,423],"of":[45,52,97,102,127,136,138,141,156,201,238,271,295,309,338,361,375,381,389,401,428],"limited":[46],"size.":[47],"However,":[48],"as":[49,248,250,364,366],"size":[51,155],"task":[54,103,129,158,168,267],"grows,":[55],"become":[59],"less":[60,63],"consistent":[61],"efficient":[64,110,279],"at":[65],"determining":[66],"useful":[68],"policy.":[69],"A":[70,289],"key":[71],"challenge":[72],"is":[76,130,184,189,243,348],"develop":[78],"methods":[79],"facilitate":[81],"scaling":[82],"up":[86],"larger,":[88],"more":[89,109],"realistic":[90,112,422],"tasks.":[91],"^":[92,213,354,432],"We":[93,214,370],"present":[94,372],"series":[96,380],"take":[100],"advantage":[101],"structure":[104,429],"make":[106,415],"display":[115,386],"such":[116],"structure.":[117],"In":[118,273,383],"each":[119,187,211,232,286,362,376],"algorithm,":[120],"we":[121,205],"assume":[122],"state":[125,142,150,207,229,251,283],"space":[126],"factored;":[131],"i.e.,":[132],"states":[133],"are":[134,430],"collections":[135],"values":[137],"set":[140],"variables.":[143],"Our":[144,323],"work":[145],"combines":[146],"hierarchical":[147,178,246],"decomposition":[148,165,179,247],"abstraction":[151,208,230,284],"reduce":[153],"prior":[159,351],"applying":[161],"learning.":[163,353,403],"Hierarchical":[164],"breaks":[166],"into":[169,268],"several":[170],"subtasks":[171,317],"be":[174],"solved":[175],"separately.":[176],"For":[177],"simplify":[181],"learning,":[182],"it":[183],"critical":[185],"subtask":[188,233],"easier":[190],"solve":[192,421],"than":[193],"overall":[195],"task.":[196],"To":[197],"achieve":[198],"goal":[200,242],"simplifying":[202],"subtasks,":[204],"perform":[206,228,282],"separately":[209],"subtask.":[212,288],"begin":[215],"presenting":[217],"algorithm":[219,255,276,291,327,363,377],"uses":[221,256],"experience":[222,341],"from":[223,340],"environment":[225],"dynamically":[227],"existing":[236],"hierarchy":[237,270],"subtasks.":[239,272,302],"Since":[240],"our":[241,395,408],"automate":[245],"well":[249,365],"abstraction,":[252],"second":[254],"dynamic":[258,334],"Bayesian":[259,335],"network":[260,336],"representation":[262,337,347],"automatically":[264],"decompose":[265],"addition,":[274],"way":[280],"resulting":[287],"third":[290],"constructs":[292],"compact":[293,304],"representations":[294,305],"activities":[296],"represent":[298],"solutions":[299,314],"These":[303],"enable":[306],"use":[308],"planning":[310],"approximate":[313],"higher-level":[316],"interacting":[319],"with":[320],"environment.":[322],"fourth":[324],"final":[326],"learn":[332],"actions":[339],"which":[345,425],"not":[349],"available":[350],"The":[355,404],"dissertation":[356],"detailed":[359],"description":[360],"some":[367],"theoretical":[368],"results.":[369],"also":[371],"experiments.":[382],"certain":[387],"types":[388,427],"structure,":[390],"simplifications":[392],"introduced":[393],"significantly":[397],"performance":[400],"provide":[410],"promising":[412],"approach":[413],"better":[418],"suited":[419],"these":[426],"present.":[431]},"counts_by_year":[{"year":2016,"cited_by_count":1},{"year":2014,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}