{"id":"https://openalex.org/W7170039563","doi":"https://doi.org/10.48550/arxiv.2607.19232","title":"S3: Stable Subgoal Selection by Constraining Uncertainty of Coarse Dynamics in Hierarchical Reinforcement Learning","display_name":"S3: Stable Subgoal Selection by Constraining Uncertainty of Coarse Dynamics in Hierarchical Reinforcement Learning","publication_year":2026,"publication_date":"2026-07-21","ids":{"openalex":"https://openalex.org/W7170039563","doi":"https://doi.org/10.48550/arxiv.2607.19232"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2607.19232","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.19232","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2607.19232","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5143413986","display_name":"Kshitij Kumar Srivastava","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Srivastava, Kshitij Kumar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5084813481","display_name":"Kshitij Jerath","orcid":"https://orcid.org/0000-0001-6356-9438"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jerath, Kshitij","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.830299973487854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.830299973487854,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.03449999913573265,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":0.00989999994635582,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7132999897003174},{"id":"https://openalex.org/keywords/selection","display_name":"Selection (genetic algorithm)","score":0.5863999724388123},{"id":"https://openalex.org/keywords/scale","display_name":"Scale (ratio)","score":0.4885999858379364},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.45100000500679016},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.33489999175071716},{"id":"https://openalex.org/keywords/system-dynamics","display_name":"System dynamics","score":0.3057999908924103}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7132999897003174},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6413000226020813},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.5863999724388123},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.546999990940094},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.4885999858379364},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.45100000500679016},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.42419999837875366},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.33489999175071716},{"id":"https://openalex.org/C77405623","wikidata":"https://www.wikidata.org/wiki/Q598451","display_name":"System dynamics","level":2,"score":0.3057999908924103},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.2827000021934509},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.28029999136924744},{"id":"https://openalex.org/C177562468","wikidata":"https://www.wikidata.org/wiki/Q182893","display_name":"Dispersion (optics)","level":2,"score":0.2623000144958496},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.25679999589920044},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.2551000118255615},{"id":"https://openalex.org/C126701199","wikidata":"https://www.wikidata.org/wiki/Q264224","display_name":"Complex dynamics","level":2,"score":0.2522999942302704},{"id":"https://openalex.org/C47822265","wikidata":"https://www.wikidata.org/wiki/Q854457","display_name":"Complex system","level":2,"score":0.2506999969482422}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2607.19232","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.19232","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2607.19232","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2607.19232","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Hierarchical":[0],"Reinforcement":[1],"Learning":[2],"(HRL)":[3],"intends":[4],"to":[5,105,134,179,185],"separate":[6],"strategic":[7],"planning":[8],"from":[9,38,52],"primitive":[10,92],"execution.":[11],"It":[12],"has":[13],"been":[14],"widely":[15],"successful":[16],"in":[17,28,35,190],"solving":[18],"long-horizon":[19,192],"and":[20,41,55,144],"complex":[21],"tasks,":[22],"where":[23],"flat-RL":[24],"algorithms":[25],"have":[26],"difficulty":[27],"learning.":[29],"However,":[30],"while":[31],"the":[32,45,53,60,73,100,117,122,129,136,141,153],"low-level":[33,61],"agent":[34,47,75,124],"HRL":[36,188],"benefits":[37],"dense":[39],"feedback":[40,51],"abundant":[42],"trial":[43],"opportunities,":[44],"high-level":[46,74,123,130],"receives":[48],"sparse,":[49],"delayed":[50],"environment":[54,110],"its":[56],"performance":[57],"depends":[58],"on":[59,91],"execution":[62],"capability.":[63],"In":[64],"this":[65],"paper,":[66],"we":[67,103,170],"study":[68],"whether":[69],"subgoal":[70,181],"selection":[71],"by":[72,81,132,156,163],"can":[76],"be":[77],"performed":[78],"more":[79],"strategically,":[80],"providing":[82],"it":[83,184],"with":[84,140],"dynamics-aware":[85,175],"intrinsic":[86,176],"motivation.":[87],"Since":[88],"motivation":[89],"based":[90],"transition":[93],"dynamics":[94],"would":[95],"require":[96],"broad":[97],"coverage":[98],"of":[99],"state-action":[101],"space,":[102],"propose":[104],"use":[106],"coarse":[107,142],"dynamics,":[108,143],"i.e.,":[109],"transitions":[111],"aggregated":[112],"over":[113],"multiple":[114],"steps":[115],"at":[116,120],"temporal":[118],"scale":[119],"which":[121],"operates.":[125],"This":[126],"approach":[127],"stabilizes":[128],"policy":[131],"learning":[133],"minimize":[135],"predictive":[137,154],"uncertainty":[138,155],"associated":[139],"provides":[145],"a":[146,164,173],"guided":[147],"structure":[148],"for":[149],"navigation.":[150],"We":[151],"model":[152],"evaluating":[157],"different":[158],"dispersion":[159],"metrics":[160],"as":[161],"approximated":[162],"Mixture":[165],"Density":[166],"Network":[167],"(MDN).":[168],"Empirically,":[169],"observe":[171],"that":[172],"dense,":[174],"reward":[177],"leads":[178],"risk-averse":[180],"selection,":[182],"enabling":[183],"outperform":[186],"state-of-the-art":[187],"methods":[189],"non-stationary":[191],"environments.":[193]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-23T00:00:00"}