{"id":"https://openalex.org/W2099860458","doi":"https://doi.org/10.1162/neco_a_00246","title":"MOSAIC for Multiple-Reward Environments","display_name":"MOSAIC for Multiple-Reward Environments","publication_year":2011,"publication_date":"2011-12-14","ids":{"openalex":"https://openalex.org/W2099860458","doi":"https://doi.org/10.1162/neco_a_00246","mag":"2099860458","pmid":"https://pubmed.ncbi.nlm.nih.gov/22168558"},"language":"en","primary_location":{"id":"doi:10.1162/neco_a_00246","is_oa":false,"landing_page_url":"https://doi.org/10.1162/neco_a_00246","pdf_url":null,"source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},"type":"article","indexed_in":["crossref","pubmed"],"open_access":{"is_oa":false,"oa_status":"closed","oa_url":null,"any_repository_has_fulltext":false},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5067055140","display_name":"Norikazu Sugimoto","orcid":null},"institutions":[{"id":"https://openalex.org/I90023481","display_name":"National Institute of Information and Communications Technology","ror":"https://ror.org/016bgq349","country_code":"JP","type":"facility","lineage":["https://openalex.org/I90023481"]}],"countries":["JP"],"is_corresponding":true,"raw_author_name":"Norikazu Sugimoto","raw_affiliation_strings":["Center for Information and Neural Networks, National Institute of Information and Communications Technology, Kyoto 619-0288, Japan, and Department of Brain Robot Interface, Brain Information Communication Research Laboratory Group, ATR, Kyoto 619-0288, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Information and Neural Networks, National Institute of Information and Communications Technology, Kyoto 619-0288, Japan, and Department of Brain Robot Interface, Brain Information Communication Research Laboratory Group, ATR, Kyoto 619-0288, Japan","institution_ids":["https://openalex.org/I90023481"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5030912694","display_name":"Masahiko Haruno","orcid":"https://orcid.org/0000-0002-0817-0720"},"institutions":[{"id":"https://openalex.org/I149481174","display_name":"Tamagawa University","ror":"https://ror.org/05f8a4p63","country_code":"JP","type":"education","lineage":["https://openalex.org/I149481174"]},{"id":"https://openalex.org/I4210086780","display_name":"Japan Science and Technology Agency","ror":"https://ror.org/00097mb19","country_code":"JP","type":"government","lineage":["https://openalex.org/I4210086780"]},{"id":"https://openalex.org/I90023481","display_name":"National Institute of Information and Communications Technology","ror":"https://ror.org/016bgq349","country_code":"JP","type":"facility","lineage":["https://openalex.org/I90023481"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Masahiko Haruno","raw_affiliation_strings":["Center for Information and Neural Networks, National Institute of Information and Communications Technology, Kyoto 619-0288, Japan, PRESTO, Japan Science and Technology Agency, Saitama 3312-0012, Japan, and Tamagawa University Brain Science Institute, Tokyo 194-8610, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Center for Information and Neural Networks, National Institute of Information and Communications Technology, Kyoto 619-0288, Japan, PRESTO, Japan Science and Technology Agency, Saitama 3312-0012, Japan, and Tamagawa University Brain Science Institute, Tokyo 194-8610, Japan","institution_ids":["https://openalex.org/I90023481","https://openalex.org/I149481174","https://openalex.org/I4210086780"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5004840638","display_name":"Kenji Doya","orcid":"https://orcid.org/0000-0002-2446-6820"},"institutions":[{"id":"https://openalex.org/I142637625","display_name":"Okinawa Institute of Science and Technology Graduate University","ror":"https://ror.org/02qg15b79","country_code":"JP","type":"education","lineage":["https://openalex.org/I142637625"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Kenji Doya","raw_affiliation_strings":["Neural Computation Unit, Okinawa Institute of Science and Technology, Okinawa 904-0495, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Neural Computation Unit, Okinawa Institute of Science and Technology, Okinawa 904-0495, Japan","institution_ids":["https://openalex.org/I142637625"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5028840283","display_name":"Mitsuo Kawato","orcid":"https://orcid.org/0000-0001-8185-1197"},"institutions":[{"id":"https://openalex.org/I4210158934","display_name":"Research Organization of Information and Systems","ror":"https://ror.org/04p4e8t29","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Mitsuo Kawato","raw_affiliation_strings":["Brain Information Communication Research Laboratory Group, ATR, Kyoto 619-0288, Japan"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brain Information Communication Research Laboratory Group, ATR, Kyoto 619-0288, Japan","institution_ids":["https://openalex.org/I4210158934"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":4,"corresponding_author_ids":["https://openalex.org/A5067055140"],"corresponding_institution_ids":["https://openalex.org/I90023481"],"apc_list":null,"apc_paid":null,"fwci":1.3187,"has_fulltext":false,"cited_by_count":29,"citation_normalized_percentile":{"value":0.84561219,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":90,"max":98},"biblio":{"volume":"24","issue":"3","first_page":"577","last_page":"606"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.996399998664856,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10982","display_name":"Motor Control and Adaptation","score":0.9866999983787537,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10653","display_name":"Robot Manipulation and Learning","score":0.9837999939918518,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6915575265884399},{"id":"https://openalex.org/keywords/controller","display_name":"Controller (irrigation)","score":0.6474758982658386},{"id":"https://openalex.org/keywords/unobservable","display_name":"Unobservable","score":0.5828391313552856},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.5701642036437988},{"id":"https://openalex.org/keywords/robot","display_name":"Robot","score":0.5490924119949341},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4832335412502289},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.46331262588500977},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4416944086551666},{"id":"https://openalex.org/keywords/dynamics","display_name":"Dynamics (music)","score":0.4182566702365875},{"id":"https://openalex.org/keywords/mathematics","display_name":"Mathematics","score":0.14067575335502625},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.13793179392814636},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.1066913902759552}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6915575265884399},{"id":"https://openalex.org/C203479927","wikidata":"https://www.wikidata.org/wiki/Q5165939","display_name":"Controller (irrigation)","level":2,"score":0.6474758982658386},{"id":"https://openalex.org/C2780695315","wikidata":"https://www.wikidata.org/wiki/Q3799040","display_name":"Unobservable","level":2,"score":0.5828391313552856},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5701642036437988},{"id":"https://openalex.org/C90509273","wikidata":"https://www.wikidata.org/wiki/Q11012","display_name":"Robot","level":2,"score":0.5490924119949341},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4832335412502289},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.46331262588500977},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4416944086551666},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.4182566702365875},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.14067575335502625},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.13793179392814636},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.1066913902759552},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C6557445","wikidata":"https://www.wikidata.org/wiki/Q173113","display_name":"Agronomy","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C19417346","wikidata":"https://www.wikidata.org/wiki/Q7922","display_name":"Pedagogy","level":1,"score":0.0},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.0}],"mesh":[{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D000465","descriptor_name":"Algorithms","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D001185","descriptor_name":"Artificial Intelligence","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D007858","descriptor_name":"Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007858","descriptor_name":"Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D007858","descriptor_name":"Learning","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012054","descriptor_name":"Reinforcement, Psychology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012054","descriptor_name":"Reinforcement, Psychology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012054","descriptor_name":"Reinforcement, Psychology","qualifier_ui":null,"qualifier_name":null,"is_major_topic":false},{"descriptor_ui":"D012201","descriptor_name":"Reward","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012201","descriptor_name":"Reward","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012201","descriptor_name":"Reward","qualifier_ui":null,"qualifier_name":null,"is_major_topic":true},{"descriptor_ui":"D012371","descriptor_name":"Robotics","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D012371","descriptor_name":"Robotics","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false},{"descriptor_ui":"D012371","descriptor_name":"Robotics","qualifier_ui":"Q000379","qualifier_name":"methods","is_major_topic":false}],"locations_count":2,"locations":[{"id":"doi:10.1162/neco_a_00246","is_oa":false,"landing_page_url":"https://doi.org/10.1162/neco_a_00246","pdf_url":null,"source":{"id":"https://openalex.org/S207023548","display_name":"Neural Computation","issn_l":"0899-7667","issn":["0899-7667","1530-888X"],"is_oa":false,"is_in_doaj":false,"is_core":true,"host_organization":"https://openalex.org/P4310315718","host_organization_name":"The MIT Press","host_organization_lineage":["https://openalex.org/P4310315718"],"host_organization_lineage_names":["The MIT Press"],"type":"journal"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural Computation","raw_type":"journal-article"},{"id":"pmid:22168558","is_oa":false,"landing_page_url":"https://pubmed.ncbi.nlm.nih.gov/22168558","pdf_url":null,"source":{"id":"https://openalex.org/S4306525036","display_name":"PubMed","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I1299303238","host_organization_name":"National Institutes of Health","host_organization_lineage":["https://openalex.org/I1299303238"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Neural computation","raw_type":null}],"best_oa_location":null,"sustainable_development_goals":[],"awards":[],"funders":[{"id":"https://openalex.org/F4320320912","display_name":"Ministry of Education, Culture, Sports, Science and Technology","ror":"https://ror.org/048rj2z13"},{"id":"https://openalex.org/F4320338111","display_name":"Precursory Research for Embryonic Science and Technology","ror":null}],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":53,"referenced_works":["https://openalex.org/W73143588","https://openalex.org/W108187717","https://openalex.org/W1486632395","https://openalex.org/W1488730473","https://openalex.org/W1494114146","https://openalex.org/W1526807135","https://openalex.org/W1585861384","https://openalex.org/W1592847719","https://openalex.org/W1603565927","https://openalex.org/W1631187438","https://openalex.org/W1798187755","https://openalex.org/W1973615355","https://openalex.org/W1998649829","https://openalex.org/W1999642580","https://openalex.org/W2043385819","https://openalex.org/W2046814205","https://openalex.org/W2069913713","https://openalex.org/W2081864293","https://openalex.org/W2084462975","https://openalex.org/W2085658907","https://openalex.org/W2095871359","https://openalex.org/W2097815751","https://openalex.org/W2097861969","https://openalex.org/W2107628283","https://openalex.org/W2107726111","https://openalex.org/W2109703216","https://openalex.org/W2109910161","https://openalex.org/W2110862341","https://openalex.org/W2113501460","https://openalex.org/W2114451917","https://openalex.org/W2115356364","https://openalex.org/W2115524942","https://openalex.org/W2120360759","https://openalex.org/W2122775046","https://openalex.org/W2129103433","https://openalex.org/W2133859362","https://openalex.org/W2136202932","https://openalex.org/W2137547873","https://openalex.org/W2139612737","https://openalex.org/W2142789491","https://openalex.org/W2145097496","https://openalex.org/W2158548602","https://openalex.org/W2160584648","https://openalex.org/W2167647761","https://openalex.org/W2168342951","https://openalex.org/W2171277043","https://openalex.org/W2539915646","https://openalex.org/W2615497679","https://openalex.org/W2911283634","https://openalex.org/W2912185451","https://openalex.org/W2914656440","https://openalex.org/W2989068617","https://openalex.org/W4249441547"],"related_works":["https://openalex.org/W2614563012","https://openalex.org/W4293337373","https://openalex.org/W1968533609","https://openalex.org/W1984270607","https://openalex.org/W2349333549","https://openalex.org/W2116445905","https://openalex.org/W2486803649","https://openalex.org/W3140685204","https://openalex.org/W2151842462","https://openalex.org/W1541823634"],"abstract_inverted_index":{"Reinforcement":[0],"learning":[1],"(RL)":[2],"can":[3],"provide":[4],"a":[5,47,61,120,267],"basic":[6],"framework":[7,216],"for":[8,35,182,217,261],"autonomous":[9],"robots":[10],"to":[11,13,54,79,165,185],"learn":[12],"control":[14,43,183],"and":[15,37,41,51,66,97,109,145,172,180,195,219,228,230,252,276,298],"maximize":[16],"future":[17],"cumulative":[18],"rewards":[19],"in":[20,116,226],"complex":[21,32,103],"environments.":[22],"To":[23],"achieve":[24],"high":[25],"performance,":[26],"RL":[27,160,218,234,239,263,294],"controllers":[28,192,264],"must":[29,141],"consider":[30,142],"the":[31,57,72,85,94,106,117,131,202,209,238,244,247,253,271],"external":[33,95],"dynamics":[34,59,96,108,171,189,207,248,275],"movements":[36],"task":[38],"(reward":[39,151],"function)":[40],"optimize":[42],"commands.":[44],"For":[45],"example,":[46],"robot":[48,77,140],"playing":[49],"tennis":[50,62,125],"squash":[52,64],"needs":[53],"cope":[55,186],"with":[56,187,270],"different":[58],"of":[60,87,93,119,124,133,170,204,246,274,288],"or":[63,136],"racket":[65],"such":[67,167],"dynamic":[68],"environmental":[69],"factors":[70],"as":[71,115],"wind.":[73],"In":[74,153],"addition,":[75],"this":[76,154,215,289],"has":[78],"tailor":[80],"its":[81,143,146,205],"tactics":[82],"simultaneously":[83],"under":[84],"rules":[86],"either":[88],"game.":[89],"This":[90],"double":[91,168],"complexity":[92,169],"reward":[98,111,254,299],"function":[99],"sometimes":[100],"becomes":[101],"more":[102],"when":[104],"both":[105],"multiple":[107,110],"functions":[112],"switch":[113],"implicitly,":[114],"situation":[118],"real":[121],"(multi-agent)":[122],"game":[123],"where":[126,190],"one":[127],"player":[128],"cannot":[129],"observe":[130],"intention":[132],"her":[134,137],"opponents":[135],"partner.":[138],"The":[139,278],"opponent's":[144],"partner's":[147],"unobservable":[148],"behavioral":[149],"goals":[150],"function).":[152],"article,":[155],"we":[156,213],"address":[157],"how":[158],"an":[159,232],"agent":[161],"should":[162],"be":[163],"designed":[164],"handle":[166],"reward.":[173],"We":[174],"have":[175],"previously":[176],"proposed":[177],"modular":[178],"selection":[179],"identification":[181],"(MOSAIC)":[184],"nonstationary":[188],"appropriate":[191,233],"are":[193,265],"selected":[194],"learned":[196],"among":[197,293],"many":[198],"candidates":[199],"based":[200,236],"on":[201,237],"error":[203,242],"paired":[206,269],"predictor:":[208],"forward":[210,250,296],"model.":[211],"Here":[212],"extend":[214],"propose":[220],"MOSAIC-MR":[221,283],"architecture.":[222],"It":[223],"resembles":[224],"MOSAIC":[225,259],"spirit":[227],"selects":[229],"learns":[231],"controller":[235],"controller's":[240],"TD":[241],"using":[243],"errors":[245],"(the":[249],"model)":[251],"predictors.":[255,300],"Furthermore,":[256],"unlike":[257],"other":[258,285],"variants":[260],"RL,":[262],"not":[266],"priori":[268],"fixed":[272],"predictors":[273],"rewards.":[277],"simulation":[279],"results":[280],"demonstrate":[281],"that":[282],"outperforms":[284],"counterparts":[286],"because":[287],"flexible":[290],"association":[291],"ability":[292],"controllers,":[295],"models,":[297]},"counts_by_year":[{"year":2025,"cited_by_count":2},{"year":2024,"cited_by_count":2},{"year":2023,"cited_by_count":5},{"year":2022,"cited_by_count":4},{"year":2021,"cited_by_count":4},{"year":2020,"cited_by_count":4},{"year":2018,"cited_by_count":1},{"year":2017,"cited_by_count":1},{"year":2016,"cited_by_count":2},{"year":2015,"cited_by_count":1},{"year":2014,"cited_by_count":1},{"year":2012,"cited_by_count":2}],"updated_date":"2026-05-21T06:26:12.895304","created_date":"2025-10-10T00:00:00"}