{"id":"https://openalex.org/W4415799017","doi":"https://doi.org/10.52202/079017-0104","title":"Goal Reduction with Loop-Removal Accelerates RL and Models Human Brain Activity in Goal-Directed Learning","display_name":"Goal Reduction with Loop-Removal Accelerates RL and Models Human Brain Activity in Goal-Directed Learning","publication_year":2024,"publication_date":"2024-01-01","ids":{"openalex":"https://openalex.org/W4415799017","doi":"https://doi.org/10.52202/079017-0104"},"language":"en","primary_location":{"id":"doi:10.52202/079017-0104","doi":"https://doi.org/10.52202/079017-0104","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-0104","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},"volume":null,"type":"conference-paper","is_traditional":true,"indexed_in":["crossref"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://www.biorxiv.org/content/biorxiv/early/2024/03/20/2024.03.19.585826.full.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5110721035","display_name":"Huzi Cheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210119109","display_name":"Indiana University Bloomington","ror":"https://ror.org/02k40bc56","country_code":"US","type":"education","lineage":["https://openalex.org/I4210119109","https://openalex.org/I592451"]},{"id":"https://openalex.org/I592451","display_name":"Indiana University","ror":"https://ror.org/01kg8sb98","country_code":"US","type":"education","lineage":["https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Huzi Cheng","raw_affiliation_strings":["Department of Psychological and Brain Sciences, Indiana University Bloomington","Indiana University Bloomington;"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Psychological and Brain Sciences, Indiana University Bloomington","institution_ids":["https://openalex.org/I4210119109","https://openalex.org/I592451"]},{"raw_affiliation_string":"Indiana University Bloomington;","institution_ids":["https://openalex.org/I4210119109"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5102713056","display_name":"Joshua W. Brown","orcid":"https://orcid.org/0009-0008-3226-6360","observed_orcids":["https://orcid.org/0009-0008-3226-6360"]},"institutions":[{"id":"https://openalex.org/I4210119109","display_name":"Indiana University Bloomington","ror":"https://ror.org/02k40bc56","country_code":"US","type":"education","lineage":["https://openalex.org/I4210119109","https://openalex.org/I592451"]},{"id":"https://openalex.org/I592451","display_name":"Indiana University","ror":"https://ror.org/01kg8sb98","country_code":"US","type":"education","lineage":["https://openalex.org/I592451"]}],"countries":["US"],"is_corresponding":true,"raw_author_name":"Joshua W. Brown","raw_affiliation_strings":["Department of Psychological and Brain Sciences, Indiana University Bloomington","Indiana University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Department of Psychological and Brain Sciences, Indiana University Bloomington","institution_ids":["https://openalex.org/I4210119109","https://openalex.org/I592451"]},{"raw_affiliation_string":"Indiana University","institution_ids":["https://openalex.org/I592451"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5102713056"],"corresponding_institution_ids":["https://openalex.org/I4210119109","https://openalex.org/I592451"],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.0,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"3185","last_page":"3208"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10906","display_name":"AI-based Problem Solving and Planning","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11574","display_name":"Artificial Intelligence in Games","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/subgoal-generation","display_name":"subgoal generation","score":0.6579999923706055},{"id":"https://openalex.org/keywords/goal-directed-learning","display_name":"goal-directed learning","score":0.9490000009536743},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"reinforcement learning","score":0.6100000143051147},{"id":"https://openalex.org/keywords/replay-buffer","display_name":"replay buffer","score":0.6790000200271606},{"id":"https://openalex.org/keywords/human-brain-activity","display_name":"human brain activity","score":0.48399999737739563},{"id":"https://openalex.org/keywords/basal-ganglia","display_name":"basal ganglia","score":1.0},{"id":"https://openalex.org/keywords/goal-directed-behavior","display_name":"goal-directed behavior","score":0.5989999771118164},{"id":"https://openalex.org/keywords/navigation","display_name":"navigation","score":0.32899999618530273},{"id":"https://openalex.org/keywords/robotic-arm-manipulation","display_name":"robotic arm manipulation","score":0.8560000061988831},{"id":"https://openalex.org/keywords/fmri-comparison","display_name":"fMRI comparison","score":0.39800000190734863},{"id":"https://openalex.org/keywords/deep-q-learning","display_name":"deep Q-learning","score":0.6150000095367432},{"id":"https://openalex.org/keywords/soft-actor-critic","display_name":"soft actor-critic","score":0.9940000176429749}],"concepts":[{"id":"https://openalex.org/C2777670902","wikidata":"https://www.wikidata.org/wiki/Q492038","display_name":"Human brain","level":2,"score":0.5842000246047974},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4948999881744385},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.48240000009536743},{"id":"https://openalex.org/C169760540","wikidata":"https://www.wikidata.org/wiki/Q207011","display_name":"Neuroscience","level":1,"score":0.46880000829696655},{"id":"https://openalex.org/C2984127161","wikidata":"https://www.wikidata.org/wiki/Q969316","display_name":"Neural activity","level":2,"score":0.38429999351501465},{"id":"https://openalex.org/C120843803","wikidata":"https://www.wikidata.org/wiki/Q4955807","display_name":"Brain activity and meditation","level":3,"score":0.3578000068664551},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3206000030040741},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.31790000200271606},{"id":"https://openalex.org/C74254510","wikidata":"https://www.wikidata.org/wiki/Q7240508","display_name":"Premovement neuronal activity","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C2985495968","wikidata":"https://www.wikidata.org/wiki/Q422453","display_name":"Motor activity","level":2,"score":0.2328999936580658}],"mesh":[],"locations_count":2,"locations":[{"id":"doi:10.52202/079017-0104","doi":"https://doi.org/10.52202/079017-0104","is_oa":false,"landing_page_url":"https://doi.org/10.52202/079017-0104","pdf_url":null,"source":{"id":"https://openalex.org/S4363606243","display_name":"neural information processing systems","issn_l":"1049-5258","issn":["1049-5258"],"is_oa":false,"is_in_doaj":false,"is_core":true,"listed_in":["cwts-core","norway-2"],"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Advances in Neural Information Processing Systems 37","raw_type":"proceedings-article"},{"id":"doi:10.1101/2024.03.19.585826","doi":"https://doi.org/10.1101/2024.03.19.585826","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2024.03.19.585826","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2024/03/20/2024.03.19.585826.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"}],"best_oa_location":{"id":"doi:10.1101/2024.03.19.585826","doi":"https://doi.org/10.1101/2024.03.19.585826","is_oa":true,"landing_page_url":"https://doi.org/10.1101/2024.03.19.585826","pdf_url":"https://www.biorxiv.org/content/biorxiv/early/2024/03/20/2024.03.19.585826.full.pdf","source":{"id":"https://openalex.org/S4306402567","display_name":"bioRxiv (Cold Spring Harbor Laboratory)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2750212522","host_organization_name":"Cold Spring Harbor Laboratory","host_organization_lineage":["https://openalex.org/I2750212522"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"bioRxiv","raw_type":"posted-content"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415799017.pdf","grobid_xml":"https://content.openalex.org/works/W4415799017.grobid-xml"},"referenced_works_count":63,"referenced_works":["https://openalex.org/W1498436455","https://openalex.org/W1526753291","https://openalex.org/W1528119265","https://openalex.org/W1766116129","https://openalex.org/W1784227921","https://openalex.org/W1969483458","https://openalex.org/W1973158444","https://openalex.org/W1995007954","https://openalex.org/W2000214310","https://openalex.org/W2015866962","https://openalex.org/W2017766656","https://openalex.org/W2041336675","https://openalex.org/W2053576047","https://openalex.org/W2057001461","https://openalex.org/W2057307785","https://openalex.org/W2058467798","https://openalex.org/W2067754053","https://openalex.org/W2070851476","https://openalex.org/W2081287319","https://openalex.org/W2086593994","https://openalex.org/W2103768739","https://openalex.org/W2113616846","https://openalex.org/W2145339207","https://openalex.org/W2150600497","https://openalex.org/W2151137320","https://openalex.org/W2160654481","https://openalex.org/W2166514604","https://openalex.org/W2169528473","https://openalex.org/W2256669593","https://openalex.org/W2346736747","https://openalex.org/W2552737632","https://openalex.org/W2563050634","https://openalex.org/W2579277897","https://openalex.org/W2618015635","https://openalex.org/W2625874945","https://openalex.org/W2736601468","https://openalex.org/W2757064418","https://openalex.org/W2765123421","https://openalex.org/W2765811365","https://openalex.org/W2781726626","https://openalex.org/W2854445389","https://openalex.org/W2913907987","https://openalex.org/W2950885698","https://openalex.org/W2951066214","https://openalex.org/W2970479807","https://openalex.org/W2973223114","https://openalex.org/W3034680299","https://openalex.org/W3103493959","https://openalex.org/W3104751091","https://openalex.org/W3105542944","https://openalex.org/W3109943994","https://openalex.org/W3156433428","https://openalex.org/W3173049816","https://openalex.org/W3185480713","https://openalex.org/W3194112469","https://openalex.org/W3212475937","https://openalex.org/W3213063001","https://openalex.org/W4205547969","https://openalex.org/W4226265259","https://openalex.org/W4285601030","https://openalex.org/W4318219818","https://openalex.org/W4362599051","https://openalex.org/W4406207021"],"related_works":["https://openalex.org/W3007440580","https://openalex.org/W1934818383","https://openalex.org/W1992631815","https://openalex.org/W7204908488","https://openalex.org/W4405276987","https://openalex.org/W2626804490","https://openalex.org/W1830929498","https://openalex.org/W2883509078","https://openalex.org/W4385768012","https://openalex.org/W3200101246"],"abstract_inverted_index":{"Abstract":[0],"Goal-directed":[1],"planning":[2],"presents":[3],"a":[4,44,59,76,139,153,177,187,209],"challenge":[5],"for":[6,38,48,83,213],"classical":[7],"RL":[8,97,130,149],"algorithms":[9],"due":[10],"to":[11,26,126],"the":[12,15,67,81,91,127,133,165,197],"vastness":[13],"of":[14,66],"combinatorial":[16],"state":[17],"and":[18,23,54,102,111,121,168,173,191,199],"goal":[19,45,157],"spaces,":[20],"while":[21],"humans":[22,206],"animals":[24],"adapt":[25],"complex":[27],"environments,":[28],"especially":[29],"with":[30,138,170],"diverse,":[31],"non-stationary":[32],"objectives,":[33],"often":[34],"employing":[35],"intermediate":[36],"goals":[37],"long-horizon":[39],"tasks.":[40],"Here,":[41],"we":[42,181],"propose":[43],"reduction":[46,158],"mechanism":[47,159],"effectively":[49],"deriving":[50],"subgoals":[51,74],"from":[52,75],"arbitrary":[53],"distant":[55],"original":[56],"goals,":[57],"using":[58],"novel":[60],"loop-removal":[61],"technique.":[62],"1":[63],"The":[64,202],"product":[65],"method,":[68],"called":[69],"goal-reducer":[70,92,134,188],",":[71,135],"distills":[72],"high-quality":[73],"replay":[77],"buffer,":[78],"all":[79],"without":[80,142],"need":[82],"prior":[84],"global":[85],"environmental":[86],"knowledge.":[87],"Simulations":[88],"show":[89],"that":[90,205],"can":[93],"be":[94],"integrated":[95,147],"into":[96],"frameworks":[98],"like":[99],"Deep":[100],"Q-learning":[101],"Soft":[103],"Actor-Critic.":[104],"It":[105],"accelerates":[106],"performance":[107,167],"in":[108,151,176],"both":[109],"discrete":[110],"continuous":[112],"action":[113],"space":[114],"tasks,":[115],"such":[116],"as":[117],"grid":[118],"world":[119],"navigation":[120,154],"robotic":[122],"arm":[123],"manipulation,":[124],"relative":[125],"corresponding":[128,192],"standard":[129],"models.":[131],"Moreover,":[132],"when":[136],"combined":[137],"local":[140],"policy,":[141],"iterative":[143],"training,":[144],"outperforms":[145],"its":[146],"deep":[148],"counterparts":[150],"solving":[152],"task.":[155],"This":[156],"also":[160],"models":[161],"human":[162,171,193],"problem-solving.":[163],"Comparing":[164],"model\u2019s":[166],"activation":[169],"behavior":[172],"fMRI":[174],"data":[175],"treasure":[178],"hunting":[179],"task,":[180],"found":[182],"matching":[183],"representational":[184],"patterns":[185],"between":[186],"agent\u2019s":[189],"components":[190],"brain":[194],"areas,":[195],"particularly":[196],"vmPFC":[198],"basal":[200],"ganglia.":[201],"results":[203],"suggest":[204],"may":[207],"use":[208],"similar":[210],"computational":[211],"framework":[212],"goal-directed":[214],"behaviors.":[215]},"counts_by_year":[],"updated_date":"2026-11-09T13:22:03.289127","created_date":"2025-11-03T00:00:00"}