{"id":"https://openalex.org/W2402540331","doi":null,"title":"Hierarchical Reinforcement Learning using Spatio-Temporal Abstractions and Deep Neural Networks.","display_name":"Hierarchical Reinforcement Learning using Spatio-Temporal Abstractions and Deep Neural Networks.","publication_year":2016,"publication_date":"2016-05-17","ids":{"openalex":"https://openalex.org/W2402540331","mag":"2402540331"},"language":"en","primary_location":{"id":"mag:2402540331","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1605.05359.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":[],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/1605.05359.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5016085812","display_name":"Ramnandan Krishnamurthy","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ramnandan Krishnamurthy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013958846","display_name":"Aravind S. Lakshminarayanan","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Aravind S. Lakshminarayanan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026334494","display_name":"Peeyush Kumar","orcid":"https://orcid.org/0000-0002-9010-6073","observed_orcids":["https://orcid.org/0000-0002-9010-6073"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peeyush Kumar","raw_affiliation_strings":[],"raw_orcid":"https://orcid.org/0000-0002-9010-6073","affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5009374923","display_name":"Balaraman Ravindran","orcid":"https://orcid.org/0000-0002-5364-7639","observed_orcids":["https://orcid.org/0000-0002-5364-7639"]},"institutions":[{"id":"https://openalex.org/I24676775","display_name":"Indian Institute of Technology Madras","ror":"https://ror.org/03v0r5n49","country_code":"IN","type":"education","lineage":["https://openalex.org/I24676775"]}],"countries":["IN"],"is_corresponding":false,"raw_author_name":"Balaraman Ravindran","raw_affiliation_strings":["Indian Institute of Technology  Madras"],"raw_orcid":"https://orcid.org/0000-0002-5364-7639","affiliations":[{"raw_affiliation_string":"Indian Institute of Technology  Madras","institution_ids":["https://openalex.org/I24676775"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":20,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/hierarchical-reinforcement-learning","display_name":"hierarchical reinforcement learning","score":0.9990000128746033},{"id":"https://openalex.org/keywords/skill-acquisition","display_name":"skill acquisition","score":0.7670000195503235},{"id":"https://openalex.org/keywords/abstract-states","display_name":"abstract states","score":0.7570000290870667},{"id":"https://openalex.org/keywords/metastable-zone","display_name":"metastable region","score":0.8009999990463257},{"id":"https://openalex.org/keywords/dynamical-systems","display_name":"dynamical systems","score":0.8790000081062317},{"id":"https://openalex.org/keywords/pcca","display_name":"PCCA","score":0.8479999899864197},{"id":"https://openalex.org/keywords/spectral-clustering","display_name":"spectral clustering","score":0.9350000023841858},{"id":"https://openalex.org/keywords/options","display_name":"options","score":0.5989999771118164},{"id":"https://openalex.org/keywords/model-free-reinforcement-learning","display_name":"model-free reinforcement learning","score":0.7540000081062317},{"id":"https://openalex.org/keywords/video-game-playing","display_name":"video gaming","score":0.6890000104904175},{"id":"https://openalex.org/keywords/representation-learning","display_name":"representation learning","score":0.6830000281333923},{"id":"https://openalex.org/keywords/deep-neural-network","display_name":"deep neural network","score":0.8740000128746033}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8527860641479492},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7765718102455139},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.646060585975647},{"id":"https://openalex.org/C124304363","wikidata":"https://www.wikidata.org/wiki/Q673661","display_name":"Abstraction","level":2,"score":0.6428332328796387},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6058739423751831},{"id":"https://openalex.org/C136197465","wikidata":"https://www.wikidata.org/wiki/Q1729295","display_name":"Variety (cybernetics)","level":2,"score":0.5618532299995422},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.5602011680603027},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.5382124781608582},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4933587610721588},{"id":"https://openalex.org/C73555534","wikidata":"https://www.wikidata.org/wiki/Q622825","display_name":"Cluster analysis","level":2,"score":0.45914363861083984},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.44917649030685425},{"id":"https://openalex.org/C59404180","wikidata":"https://www.wikidata.org/wiki/Q17013334","display_name":"Feature learning","level":2,"score":0.41215693950653076},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.0},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.0},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.0},{"id":"https://openalex.org/C94625758","wikidata":"https://www.wikidata.org/wiki/Q7163","display_name":"Politics","level":2,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"mag:2402540331","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1605.05359.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null}],"best_oa_location":{"id":"mag:2402540331","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/pdf/1605.05359.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":27,"referenced_works":["https://openalex.org/W32403112","https://openalex.org/W59183349","https://openalex.org/W200434350","https://openalex.org/W1494114146","https://openalex.org/W1515851193","https://openalex.org/W1532854728","https://openalex.org/W1536990779","https://openalex.org/W1545006598","https://openalex.org/W1584307643","https://openalex.org/W1595483645","https://openalex.org/W1778554682","https://openalex.org/W2006533296","https://openalex.org/W2109910161","https://openalex.org/W2118688707","https://openalex.org/W2121517924","https://openalex.org/W2142838865","https://openalex.org/W2143435603","https://openalex.org/W2145339207","https://openalex.org/W2150468603","https://openalex.org/W2165874743","https://openalex.org/W2167945827","https://openalex.org/W2168640731","https://openalex.org/W2201581102","https://openalex.org/W2211996086","https://openalex.org/W2269570957","https://openalex.org/W2335959470","https://openalex.org/W2802774918"],"related_works":["https://openalex.org/W2523124567","https://openalex.org/W296226304","https://openalex.org/W7039055770","https://openalex.org/W2187328545","https://openalex.org/W4313485340","https://openalex.org/W1985756506","https://openalex.org/W2803793821","https://openalex.org/W4401157207","https://openalex.org/W57282082","https://openalex.org/W1544120923"],"abstract_inverted_index":{"This":[0],"paper":[1],"introduces":[2],"an":[3],"automated":[4,190],"skill":[5],"acquisition":[6,192],"framework":[7,193],"in":[8,21,36,76,105],"reinforcement":[9,46],"learning":[10,47,48,133,199,208,212],"which":[11],"involves":[12],"identifying":[13],"a":[14,141,147,163],"hierarchical":[15],"description":[16],"of":[17,23,107,131,143,154,166],"the":[18,37,77,99,108,132,155,167,175,189],"given":[19],"task":[20,38,134],"terms":[22,106],"abstract":[24,30,84,112],"states":[25],"and":[26,43,80,135,169],"extended":[27],"actions":[28],"between":[29,110],"states.":[31,85,113],"Identifying":[32],"such":[33,56,111,197],"structures":[34,51],"present":[35,184],"provides":[39],"ways":[40],"to":[41,54,72,93,98,121,187,194,200,213],"simplify":[42],"speed":[44],"up":[45],"algorithms.":[49],"These":[50],"also":[52],"help":[53],"generalize":[55],"algorithms":[57],"over":[58,146],"multiple":[59],"tasks":[60,144,196],"without":[61],"relearning":[62],"policies":[63],"from":[64,69,117,179],"scratch.":[65],"We":[66],"use":[67,206],"ideas":[68],"dynamical":[70],"systems":[71],"find":[73],"metastable":[74],"regions":[75],"state":[78,149],"space":[79],"associate":[81],"them":[82],"with":[83],"The":[86,114,127],"spectral":[87],"clustering":[88],"algorithm":[89],"PCCA+":[90,118],"is":[91,119,157],"used":[92,120],"identify":[94],"suitable":[95],"abstractions":[96],"aligned":[97],"underlying":[100],"structure.":[101],"Skills":[102],"are":[103,129,177],"defined":[104,145],"transitions":[109],"connectivity":[115],"information":[116],"generate":[122],"these":[123],"skills":[124,128,191],"or":[125],"options.":[126],"independent":[130],"can":[136,170],"be":[137],"efficiently":[138],"reused":[139],"across":[140],"variety":[142],"common":[148],"space.":[150],"Another":[151],"major":[152],"advantage":[153],"approach":[156],"that":[158],"it":[159],"does":[160],"not":[161],"need":[162],"prior":[164],"model":[165],"MDP":[168],"work":[171],"well":[172],"even":[173],"when":[174],"MDPs":[176],"constructed":[178],"sampled":[180],"trajectories.":[181],"Finally,":[182],"we":[183,205],"our":[185,215],"attempts":[186],"extend":[188],"complex":[195],"as":[198],"play":[201],"video":[202],"games":[203],"where":[204],"deep":[207],"techniques":[209],"for":[210],"representation":[211],"aid":[214],"spatio-temporal":[216],"abstraction":[217],"framework.":[218]},"counts_by_year":[{"year":2024,"cited_by_count":1},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1},{"year":2020,"cited_by_count":1},{"year":2019,"cited_by_count":5},{"year":2018,"cited_by_count":6},{"year":2017,"cited_by_count":3},{"year":2016,"cited_by_count":2}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2016-06-24T00:00:00"}