{"id":"https://openalex.org/W3020949915","doi":"https://doi.org/10.24963/ijcai.2020/390","title":"Reward Prediction Error as an Exploration Objective in Deep RL","display_name":"Reward Prediction Error as an Exploration Objective in Deep RL","publication_year":2020,"publication_date":"2020-07-01","ids":{"openalex":"https://openalex.org/W3020949915","doi":"https://doi.org/10.24963/ijcai.2020/390","mag":"3020949915"},"language":"en","primary_location":{"id":"doi:10.24963/ijcai.2020/390","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2020/390","pdf_url":"https://www.ijcai.org/proceedings/2020/0390.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"gold","oa_url":"https://www.ijcai.org/proceedings/2020/0390.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047752767","display_name":"Riley Simmons-Edler","orcid":null},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]},{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR","US"],"is_corresponding":false,"raw_author_name":"Riley Simmons-Edler","raw_affiliation_strings":["Princeton University","Samsung AI Center NYC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton University","institution_ids":["https://openalex.org/I20089843"]},{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5039692309","display_name":"Ben Eisner","orcid":null},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Ben Eisner","raw_affiliation_strings":["Samsung AI Center NYC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102830923","display_name":"Daniel Yang","orcid":"https://orcid.org/0000-0003-0532-1455"},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Daniel Yang","raw_affiliation_strings":["Samsung AI Center NYC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067140032","display_name":"Anthony Bisulco","orcid":"https://orcid.org/0000-0003-4418-7172"},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Anthony Bisulco","raw_affiliation_strings":["Samsung AI Center NYC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5074009646","display_name":"Eric Mitchell","orcid":"https://orcid.org/0000-0002-7487-1744"},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]},{"id":"https://openalex.org/I97018004","display_name":"Stanford University","ror":"https://ror.org/00f54p054","country_code":"US","type":"education","lineage":["https://openalex.org/I97018004"]}],"countries":["KR","US"],"is_corresponding":false,"raw_author_name":"Eric Mitchell","raw_affiliation_strings":["Samsung AI Center NYC","Stanford University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]},{"raw_affiliation_string":"Stanford University","institution_ids":["https://openalex.org/I97018004"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5091146918","display_name":"Sebastian Seung","orcid":null},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Sebastian Seung","raw_affiliation_strings":["Samsung AI Center NYC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5100392980","display_name":"Daniel Lee","orcid":"https://orcid.org/0000-0001-9447-4175"},"institutions":[{"id":"https://openalex.org/I2250650973","display_name":"Samsung (South Korea)","ror":"https://ror.org/04w3jy968","country_code":"KR","type":"company","lineage":["https://openalex.org/I2250650973"]}],"countries":["KR"],"is_corresponding":false,"raw_author_name":"Daniel Lee","raw_affiliation_strings":["Samsung AI Center NYC"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Samsung AI Center NYC","institution_ids":["https://openalex.org/I2250650973"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":3,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":"2816","last_page":"2823"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9998000264167786,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11975","display_name":"Evolutionary Algorithms and Applications","score":0.9894999861717224,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9854000210762024,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/novelty","display_name":"Novelty","score":0.866274356842041},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.8536376357078552},{"id":"https://openalex.org/keywords/dither","display_name":"Dither","score":0.7801908254623413},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.6939754486083984},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.6268371343612671},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.6186562180519104},{"id":"https://openalex.org/keywords/temporal-difference-learning","display_name":"Temporal difference learning","score":0.5973799228668213},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.5898395776748657},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.5047563314437866},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4740961492061615},{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.460897833108902},{"id":"https://openalex.org/keywords/curiosity","display_name":"Curiosity","score":0.4240129590034485},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.10792389512062073},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.08286172151565552},{"id":"https://openalex.org/keywords/computer-vision","display_name":"Computer vision","score":0.06697428226470947}],"concepts":[{"id":"https://openalex.org/C2778738651","wikidata":"https://www.wikidata.org/wiki/Q16546687","display_name":"Novelty","level":2,"score":0.866274356842041},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.8536376357078552},{"id":"https://openalex.org/C70451592","wikidata":"https://www.wikidata.org/wiki/Q376493","display_name":"Dither","level":3,"score":0.7801908254623413},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6939754486083984},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.6268371343612671},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.6186562180519104},{"id":"https://openalex.org/C196340769","wikidata":"https://www.wikidata.org/wiki/Q7698910","display_name":"Temporal difference learning","level":3,"score":0.5973799228668213},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.5898395776748657},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.5047563314437866},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4740961492061615},{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.460897833108902},{"id":"https://openalex.org/C33435437","wikidata":"https://www.wikidata.org/wiki/Q366791","display_name":"Curiosity","level":2,"score":0.4240129590034485},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.10792389512062073},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.08286172151565552},{"id":"https://openalex.org/C31972630","wikidata":"https://www.wikidata.org/wiki/Q844240","display_name":"Computer vision","level":1,"score":0.06697428226470947},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.0},{"id":"https://openalex.org/C201995342","wikidata":"https://www.wikidata.org/wiki/Q682496","display_name":"Systems engineering","level":1,"score":0.0},{"id":"https://openalex.org/C127313418","wikidata":"https://www.wikidata.org/wiki/Q1069","display_name":"Geology","level":0,"score":0.0},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.0},{"id":"https://openalex.org/C86803240","wikidata":"https://www.wikidata.org/wiki/Q420","display_name":"Biology","level":0,"score":0.0},{"id":"https://openalex.org/C111368507","wikidata":"https://www.wikidata.org/wiki/Q43518","display_name":"Oceanography","level":1,"score":0.0},{"id":"https://openalex.org/C9083635","wikidata":"https://www.wikidata.org/wiki/Q2133535","display_name":"Noise shaping","level":2,"score":0.0},{"id":"https://openalex.org/C78458016","wikidata":"https://www.wikidata.org/wiki/Q840400","display_name":"Evolutionary biology","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.24963/ijcai.2020/390","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2020/390","pdf_url":"https://www.ijcai.org/proceedings/2020/0390.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:1906.08189","is_oa":true,"landing_page_url":"http://arxiv.org/abs/1906.08189","pdf_url":"https://arxiv.org/pdf/1906.08189","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"","raw_type":"text"},{"id":"mag:3020949915","is_oa":true,"landing_page_url":"http://export.arxiv.org/pdf/1906.08189","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.1906.08189","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.1906.08189","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.24963/ijcai.2020/390","is_oa":true,"landing_page_url":"https://doi.org/10.24963/ijcai.2020/390","pdf_url":"https://www.ijcai.org/proceedings/2020/0390.pdf","source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the Twenty-Ninth International Joint Conference on Artificial Intelligence","raw_type":"proceedings-article"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3020949915.pdf","grobid_xml":"https://content.openalex.org/works/W3020949915.grobid-xml"},"referenced_works_count":21,"referenced_works":["https://openalex.org/W1771410628","https://openalex.org/W2106639887","https://openalex.org/W2121863487","https://openalex.org/W2145339207","https://openalex.org/W2169619645","https://openalex.org/W2173564293","https://openalex.org/W2561776174","https://openalex.org/W2604373826","https://openalex.org/W2751973545","https://openalex.org/W2785542505","https://openalex.org/W2886140408","https://openalex.org/W2913343212","https://openalex.org/W2924205497","https://openalex.org/W2962723954","https://openalex.org/W2963639957","https://openalex.org/W2963923407","https://openalex.org/W2964001908","https://openalex.org/W2964067469","https://openalex.org/W2964336776","https://openalex.org/W2996145485","https://openalex.org/W3103780890"],"related_works":["https://openalex.org/W3034853386","https://openalex.org/W2952154767","https://openalex.org/W3118606634","https://openalex.org/W2970868077","https://openalex.org/W2986020463","https://openalex.org/W3092156990","https://openalex.org/W2897200624","https://openalex.org/W3163453895","https://openalex.org/W3035599863","https://openalex.org/W3130271297","https://openalex.org/W3198608914","https://openalex.org/W3133533407","https://openalex.org/W3212897811","https://openalex.org/W2890882194","https://openalex.org/W3200112562","https://openalex.org/W3098951764","https://openalex.org/W2605369401","https://openalex.org/W3204272549","https://openalex.org/W3099159407","https://openalex.org/W3181335123"],"abstract_inverted_index":{"A":[0],"major":[1],"challenge":[2],"in":[3,70,86,92,155,188],"reinforcement":[4,137],"learning":[5,138],"is":[6,178,199],"exploration,":[7],"when":[8],"local":[9,108],"dithering":[10],"methods":[11,25,48],"such":[12,102],"as":[13,103],"epsilon-greedy":[14,68],"sampling":[15],"are":[16,49,75],"insufficient":[17],"to":[18,28,33,40,128,150,180],"solve":[19,151],"a":[20,113,124,135,148,184],"given":[21],"task.":[22],"Many":[23],"recent":[24],"have":[26],"proposed":[27],"intrinsically":[29],"motivate":[30],"an":[31],"agent":[32,39],"seek":[34],"novel":[35,54,89],"states,":[36],"driving":[37],"the":[38,73,118,143,160,192],"discover":[41],"improved":[42,59,203],"reward.":[43,131,204],"However,":[44],"while":[45],"state-novelty":[46,186],"exploration":[47,84,115,153,161],"suitable":[50],"for":[51],"tasks":[52,85,105,154,170,195],"where":[53,72,98,196],"observations":[55],"correlate":[56],"well":[57],"with":[58,202],"reward,":[60,95],"they":[61],"may":[62],"not":[63,76,200],"explore":[64],"more":[65],"efficiently":[66],"than":[67,183],"approaches":[69],"environments":[71],"two":[74],"well-correlated.":[77],"In":[78],"this":[79],"paper,":[80],"we":[81],"distinguish":[82],"between":[83],"which":[87,141],"seeking":[88],"states":[90],"aids":[91],"finding":[93],"new":[94,114],"and":[96,106,171,174],"those":[97],"it":[99],"does":[100],"not,":[101],"goal-conditioned":[104],"escaping":[107],"reward":[109,119],"maxima.":[110],"We":[111,132,158],"propose":[112,134],"objective,":[116],"maximizing":[117],"prediction":[120],"error":[121,146],"(RPE)":[122],"of":[123,147,163],"value":[125],"function":[126],"trained":[127],"predict":[129],"extrinsic":[130],"then":[133],"deep":[136],"method,":[139],"QXplore,":[140],"exploits":[142],"temporal":[144],"difference":[145],"Q-function":[149],"hard":[152],"high-dimensional":[156],"MDPs.":[157],"demonstrate":[159],"behavior":[162],"QXplore":[164,177],"on":[165,194],"several":[166],"OpenAI":[167],"Gym":[168],"MuJoCo":[169],"Atari":[172],"games":[173],"observe":[175],"that":[176],"comparable":[179],"or":[181],"better":[182],"baseline":[185,193],"method":[187],"all":[189],"cases,":[190],"outperforming":[191],"state":[197],"novelty":[198],"well-correlated":[201]},"counts_by_year":[{"year":2025,"cited_by_count":1}],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}