{"id":"https://openalex.org/W3200024664","doi":"https://doi.org/10.1109/ijcnn52387.2021.9533401","title":"Reward Shaping with Dynamic Trajectory Aggregation","display_name":"Reward Shaping with Dynamic Trajectory Aggregation","publication_year":2021,"publication_date":"2021-07-18","ids":{"openalex":"https://openalex.org/W3200024664","doi":"https://doi.org/10.1109/ijcnn52387.2021.9533401","mag":"3200024664"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn52387.2021.9533401","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn52387.2021.9533401","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"article","indexed_in":["crossref"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057299724","display_name":"Takato Okudo","orcid":"https://orcid.org/0000-0002-7218-7842"},"institutions":[{"id":"https://openalex.org/I200475212","display_name":"The Graduate University for Advanced Studies, SOKENDAI","ror":"https://ror.org/0516ah480","country_code":"JP","type":"education","lineage":["https://openalex.org/I200475212"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Takato Okudo","raw_affiliation_strings":["The Graduate University for Advanced Studies, SOKENDAI, Tokyo, Japna"],"affiliations":[{"raw_affiliation_string":"The Graduate University for Advanced Studies, SOKENDAI, Tokyo, Japna","institution_ids":["https://openalex.org/I200475212"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101954161","display_name":"Seiji Yamada","orcid":"https://orcid.org/0000-0002-5907-7382"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Seiji Yamada","raw_affiliation_strings":["National Institute of Informatice, NII, Tokyo, Japan"],"affiliations":[{"raw_affiliation_string":"National Institute of Informatice, NII, Tokyo, Japan","institution_ids":["https://openalex.org/I184597095"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":["https://openalex.org/A5057299724"],"corresponding_institution_ids":["https://openalex.org/I200475212"],"apc_list":null,"apc_paid":null,"fwci":0.2719,"has_fulltext":false,"cited_by_count":3,"citation_normalized_percentile":{"value":0.63425866,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":95},"biblio":{"volume":"70","issue":null,"first_page":"1","last_page":"9"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994000196456909,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9713000059127808,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10260","display_name":"Software Engineering Research","score":0.9707000255584717,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.71555095911026},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6947898864746094},{"id":"https://openalex.org/keywords/bottleneck","display_name":"Bottleneck","score":0.6677261590957642},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.6052190661430359},{"id":"https://openalex.org/keywords/trajectory","display_name":"Trajectory","score":0.5597100853919983},{"id":"https://openalex.org/keywords/bellman-equation","display_name":"Bellman equation","score":0.5184928178787231},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4835048019886017},{"id":"https://openalex.org/keywords/state-space","display_name":"State space","score":0.47206881642341614},{"id":"https://openalex.org/keywords/action","display_name":"Action (physics)","score":0.4539317190647125},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.4410480260848999},{"id":"https://openalex.org/keywords/series","display_name":"Series (stratigraphy)","score":0.43169063329696655},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.3482632040977478},{"id":"https://openalex.org/keywords/mathematical-optimization","display_name":"Mathematical optimization","score":0.26868143677711487},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.1786574423313141}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.71555095911026},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6947898864746094},{"id":"https://openalex.org/C2780513914","wikidata":"https://www.wikidata.org/wiki/Q18210350","display_name":"Bottleneck","level":2,"score":0.6677261590957642},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.6052190661430359},{"id":"https://openalex.org/C13662910","wikidata":"https://www.wikidata.org/wiki/Q193139","display_name":"Trajectory","level":2,"score":0.5597100853919983},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.5184928178787231},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4835048019886017},{"id":"https://openalex.org/C72434380","wikidata":"https://www.wikidata.org/wiki/Q230930","display_name":"State space","level":2,"score":0.47206881642341614},{"id":"https://openalex.org/C2780791683","wikidata":"https://www.wikidata.org/wiki/Q846785","display_name":"Action (physics)","level":2,"score":0.4539317190647125},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.4410480260848999},{"id":"https://openalex.org/C143724316","wikidata":"https://www.wikidata.org/wiki/Q312468","display_name":"Series (stratigraphy)","level":2,"score":0.43169063329696655},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3482632040977478},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.26868143677711487},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.1786574423313141},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.0},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.0},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.0},{"id":"https://openalex.org/C62520636","wikidata":"https://www.wikidata.org/wiki/Q944","display_name":"Quantum mechanics","level":1,"score":0.0},{"id":"https://openalex.org/C151730666","wikidata":"https://www.wikidata.org/wiki/Q7205","display_name":"Paleontology","level":1,"score":0.0},{"id":"https://openalex.org/C111919701","wikidata":"https://www.wikidata.org/wiki/Q9135","display_name":"Operating system","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C1276947","wikidata":"https://www.wikidata.org/wiki/Q333","display_name":"Astronomy","level":1,"score":0.0}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.1109/ijcnn52387.2021.9533401","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn52387.2021.9533401","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":57,"referenced_works":["https://openalex.org/W1501005121","https://openalex.org/W1553476745","https://openalex.org/W1777239053","https://openalex.org/W1999874108","https://openalex.org/W2061562262","https://openalex.org/W2097381042","https://openalex.org/W2098774185","https://openalex.org/W2106261932","https://openalex.org/W2108535023","https://openalex.org/W2109910161","https://openalex.org/W2121863487","https://openalex.org/W2130750514","https://openalex.org/W2151382427","https://openalex.org/W2156869222","https://openalex.org/W2161009228","https://openalex.org/W2164419340","https://openalex.org/W2202549229","https://openalex.org/W2218252352","https://openalex.org/W2293184044","https://openalex.org/W2594829461","https://openalex.org/W2789008106","https://openalex.org/W2792217087","https://openalex.org/W2837981523","https://openalex.org/W2944766483","https://openalex.org/W2945317616","https://openalex.org/W2949267040","https://openalex.org/W2962715211","https://openalex.org/W2963277051","https://openalex.org/W2963864421","https://openalex.org/W2964227312","https://openalex.org/W2970479807","https://openalex.org/W2971266180","https://openalex.org/W2984187004","https://openalex.org/W3037207827","https://openalex.org/W3103379718","https://openalex.org/W4214717370","https://openalex.org/W4288029578","https://openalex.org/W4288093623","https://openalex.org/W4293872189","https://openalex.org/W4300198501","https://openalex.org/W4402843978","https://openalex.org/W6638088447","https://openalex.org/W6674600207","https://openalex.org/W6674884181","https://openalex.org/W6676072908","https://openalex.org/W6676557315","https://openalex.org/W6682205418","https://openalex.org/W6684159546","https://openalex.org/W6684921986","https://openalex.org/W6689158983","https://openalex.org/W6718092244","https://openalex.org/W6727349600","https://openalex.org/W6734215269","https://openalex.org/W6751955673","https://openalex.org/W6762729159","https://openalex.org/W6767317771","https://openalex.org/W6767862010"],"related_works":["https://openalex.org/W2386410636","https://openalex.org/W2025663273","https://openalex.org/W3038962357","https://openalex.org/W4225571923","https://openalex.org/W3212257828","https://openalex.org/W2999580272","https://openalex.org/W4313679781","https://openalex.org/W3099153698","https://openalex.org/W4297873223","https://openalex.org/W2350784623"],"abstract_inverted_index":{"Reinforcement":[0],"learning,":[1],"which":[2],"acquires":[3,87],"a":[4,48,60,65,130,203,221],"policy":[5],"maximizing":[6],"long-term":[7],"rewards,":[8],"has":[9],"been":[10],"actively":[11],"studied.":[12],"Unfortunately,":[13],"this":[14,102],"learning":[15,40,206,231],"type":[16],"is":[17,42,47,56,72,104,115],"too":[18],"slow":[19],"and":[20,86,147,154,163,189,193,195,208,215],"difficult":[21,74],"to":[22,58,75,95,109,120,166],"use":[23],"in":[24,33,142,184,230],"practical":[25],"situations":[26],"because":[27],"the":[28,77,83,96,105,151,164,182],"state-action":[29],"space":[30],"becomes":[31],"huge":[32],"real":[34],"environments.":[35],"The":[36,99],"essential":[37],"factor":[38],"for":[39,51,68,118,125,178],"efficiency":[41],"rewards.":[43,53],"Potential-based":[44],"reward":[45,218,224],"shaping":[46,225],"basic":[49],"method":[50,55,103,138,201],"enriching":[52],"This":[54,137],"required":[57],"define":[59],"specific":[61],"real-value":[62],"function":[63,79,85,124],"called":[64],"\u201cpotential":[66],"function\u201d":[67],"every":[69],"domain.":[70],"It":[71,158],"often":[73],"represent":[76],"potential":[78,84],"directly.":[80],"SARSA-RS":[81,90],"learns":[82],"it.":[88],"However,":[89],"can":[91],"only":[92,150],"be":[93],"applied":[94],"simple":[97],"environment.":[98],"bottleneck":[100],"of":[101,107],"aggregation":[106,123,132],"states":[108,112,141,188],"make":[110],"abstract":[111],"since":[113],"it":[114],"almost":[116],"impossible":[117],"designers":[119],"build":[121],"an":[122,143],"all":[126,227],"states.":[127],"We":[128,172,180,198],"propose":[129],"trajectory":[131],"that":[133],"uses":[134],"subgoal":[135,152,155,174,214],"series.":[136],"dynamically":[139],"aggregates":[140],"episode":[144],"during":[145],"trial":[146],"error":[148],"with":[149,168,202],"series":[153,175],"identification":[156],"function.":[157],"makes":[159],"designer":[160],"effort":[161],"minimal":[162],"application":[165],"environments":[167],"high-dimensional":[169],"observations":[170],"possible.":[171],"obtained":[173],"from":[176],"participants":[177],"experiments.":[179],"conducted":[181],"experiments":[183],"three":[185],"domains,":[186],"four-rooms(discrete":[187],"discrete":[190],"actions),":[191],"pinball(continuous":[192],"discrete),":[194],"picking(both":[196],"continuous).":[197],"compared":[199],"our":[200,223],"baseline":[204],"reinforcement":[205],"algorithm":[207],"other":[209,228],"subgoal-based":[210,217],"methods,":[211],"including":[212],"random":[213],"naive":[216],"shaping.":[219],"As":[220],"result,":[222],"outperformed":[226],"methods":[229],"efficiency.":[232]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2025-11-06T03:46:38.306776","created_date":"2025-10-10T00:00:00"}