{"id":"https://openalex.org/W3110050486","doi":"https://doi.org/10.1109/ijcnn52387.2021.9534243","title":"Predictive PER: Balancing Priority and Diversity Towards Stable Deep Reinforcement Learning","display_name":"Predictive PER: Balancing Priority and Diversity Towards Stable Deep Reinforcement Learning","publication_year":2021,"publication_date":"2021-07-18","ids":{"openalex":"https://openalex.org/W3110050486","doi":"https://doi.org/10.1109/ijcnn52387.2021.9534243","mag":"3110050486"},"language":"en","primary_location":{"id":"doi:10.1109/ijcnn52387.2021.9534243","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn52387.2021.9534243","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2011.13093","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100750930","display_name":"Sang-Hwa Lee","orcid":"https://orcid.org/0000-0002-1114-8786"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Sanghwa Lee","raw_affiliation_strings":["National Institute of Informatics, SOKENDAI,Tokyo,Japan","National Institute of Informatics/Sokendai, Tokyo, Japan#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Informatics, SOKENDAI,Tokyo,Japan","institution_ids":["https://openalex.org/I184597095"]},{"raw_affiliation_string":"National Institute of Informatics/Sokendai, Tokyo, Japan#TAB#","institution_ids":["https://openalex.org/I184597095"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100334542","display_name":"Jaeyoung Lee","orcid":"https://orcid.org/0000-0003-4390-7676"},"institutions":[{"id":"https://openalex.org/I151746483","display_name":"University of Waterloo","ror":"https://ror.org/01aff2v68","country_code":"CA","type":"education","lineage":["https://openalex.org/I151746483"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Jaeyoung Lee","raw_affiliation_strings":["University of Waterloo,Electrical & Computer Engineering,Waterloo,Canada","University of Waterloo, Electrical & Computer Engineering, Waterloo, Canada"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Waterloo,Electrical & Computer Engineering,Waterloo,Canada","institution_ids":["https://openalex.org/I151746483"]},{"raw_affiliation_string":"University of Waterloo, Electrical & Computer Engineering, Waterloo, Canada","institution_ids":["https://openalex.org/I151746483"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5013382452","display_name":"Ichiro Hasuo","orcid":"https://orcid.org/0000-0002-8300-4650"},"institutions":[{"id":"https://openalex.org/I184597095","display_name":"National Institute of Informatics","ror":"https://ror.org/04ksd4g47","country_code":"JP","type":"facility","lineage":["https://openalex.org/I1319490839","https://openalex.org/I184597095","https://openalex.org/I4210158934"]}],"countries":["JP"],"is_corresponding":false,"raw_author_name":"Ichiro Hasuo","raw_affiliation_strings":["National Institute of Informatics, SOKENDAI,Tokyo,Japan","National Institute of Informatics/Sokendai, Tokyo, Japan#TAB#"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"National Institute of Informatics, SOKENDAI,Tokyo,Japan","institution_ids":["https://openalex.org/I184597095"]},{"raw_affiliation_string":"National Institute of Informatics/Sokendai, Tokyo, Japan#TAB#","institution_ids":["https://openalex.org/I184597095"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.1466,"has_fulltext":false,"cited_by_count":1,"citation_normalized_percentile":{"value":0.32441879,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":{"min":89,"max":93},"biblio":{"volume":"19","issue":null,"first_page":"1","last_page":"10"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9962999820709229,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10603","display_name":"Smart Grid Energy Management","score":0.9775999784469604,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9728000164031982,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/forgetting","display_name":"Forgetting","score":0.8024135828018188},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.7754640579223633},{"id":"https://openalex.org/keywords/computer-science","display_name":"Computer science","score":0.7073488235473633},{"id":"https://openalex.org/keywords/prioritization","display_name":"Prioritization","score":0.6819944381713867},{"id":"https://openalex.org/keywords/outlier","display_name":"Outlier","score":0.6348909139633179},{"id":"https://openalex.org/keywords/diversity","display_name":"Diversity (politics)","score":0.5902460217475891},{"id":"https://openalex.org/keywords/stability","display_name":"Stability (learning theory)","score":0.5590946674346924},{"id":"https://openalex.org/keywords/reinforcement","display_name":"Reinforcement","score":0.5403717756271362},{"id":"https://openalex.org/keywords/countermeasure","display_name":"Countermeasure","score":0.48070818185806274},{"id":"https://openalex.org/keywords/artificial-intelligence","display_name":"Artificial intelligence","score":0.4802544414997101},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.472510427236557},{"id":"https://openalex.org/keywords/machine-learning","display_name":"Machine learning","score":0.4154946804046631},{"id":"https://openalex.org/keywords/cognitive-psychology","display_name":"Cognitive psychology","score":0.08051526546478271},{"id":"https://openalex.org/keywords/engineering","display_name":"Engineering","score":0.0748419463634491},{"id":"https://openalex.org/keywords/law","display_name":"Law","score":0.06574904918670654},{"id":"https://openalex.org/keywords/psychology","display_name":"Psychology","score":0.0623871386051178},{"id":"https://openalex.org/keywords/social-psychology","display_name":"Social psychology","score":0.05918055772781372},{"id":"https://openalex.org/keywords/management-science","display_name":"Management science","score":0.05499923229217529}],"concepts":[{"id":"https://openalex.org/C7149132","wikidata":"https://www.wikidata.org/wiki/Q1377840","display_name":"Forgetting","level":2,"score":0.8024135828018188},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7754640579223633},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7073488235473633},{"id":"https://openalex.org/C2777615720","wikidata":"https://www.wikidata.org/wiki/Q11888847","display_name":"Prioritization","level":2,"score":0.6819944381713867},{"id":"https://openalex.org/C79337645","wikidata":"https://www.wikidata.org/wiki/Q779824","display_name":"Outlier","level":2,"score":0.6348909139633179},{"id":"https://openalex.org/C2781316041","wikidata":"https://www.wikidata.org/wiki/Q1230584","display_name":"Diversity (politics)","level":2,"score":0.5902460217475891},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.5590946674346924},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.5403717756271362},{"id":"https://openalex.org/C21593369","wikidata":"https://www.wikidata.org/wiki/Q1032176","display_name":"Countermeasure","level":2,"score":0.48070818185806274},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4802544414997101},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.472510427236557},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4154946804046631},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.08051526546478271},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.0748419463634491},{"id":"https://openalex.org/C199539241","wikidata":"https://www.wikidata.org/wiki/Q7748","display_name":"Law","level":1,"score":0.06574904918670654},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.0623871386051178},{"id":"https://openalex.org/C77805123","wikidata":"https://www.wikidata.org/wiki/Q161272","display_name":"Social psychology","level":1,"score":0.05918055772781372},{"id":"https://openalex.org/C539667460","wikidata":"https://www.wikidata.org/wiki/Q2414942","display_name":"Management science","level":1,"score":0.05499923229217529},{"id":"https://openalex.org/C43617362","wikidata":"https://www.wikidata.org/wiki/Q170050","display_name":"Chromatography","level":1,"score":0.0},{"id":"https://openalex.org/C17744445","wikidata":"https://www.wikidata.org/wiki/Q36442","display_name":"Political science","level":0,"score":0.0},{"id":"https://openalex.org/C185592680","wikidata":"https://www.wikidata.org/wiki/Q2329","display_name":"Chemistry","level":0,"score":0.0},{"id":"https://openalex.org/C146978453","wikidata":"https://www.wikidata.org/wiki/Q3798668","display_name":"Aerospace engineering","level":1,"score":0.0}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1109/ijcnn52387.2021.9534243","is_oa":false,"landing_page_url":"https://doi.org/10.1109/ijcnn52387.2021.9534243","pdf_url":null,"source":null,"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"2021 International Joint Conference on Neural Networks (IJCNN)","raw_type":"proceedings-article"},{"id":"pmh:oai:arXiv.org:2011.13093","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2011.13093","pdf_url":"https://arxiv.org/pdf/2011.13093","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"mag:3110050486","is_oa":true,"landing_page_url":"http://arxiv.org/pdf/2011.13093.pdf","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"arXiv (Cornell University)","raw_type":null},{"id":"doi:10.48550/arxiv.2011.13093","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2011.13093","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2011.13093","is_oa":true,"landing_page_url":"http://arxiv.org/abs/2011.13093","pdf_url":"https://arxiv.org/pdf/2011.13093","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"awards":[{"id":"https://openalex.org/G5850690258","display_name":null,"funder_award_id":"JPMJER1603","funder_id":"https://openalex.org/F4320320907","funder_display_name":"Japan Science and Technology Corporation"}],"funders":[{"id":"https://openalex.org/F4320320907","display_name":"Japan Science and Technology Corporation","ror":"https://ror.org/00097mb19"}],"has_content":{"pdf":true,"grobid_xml":false},"content_urls":{"pdf":"https://content.openalex.org/works/W3110050486.pdf"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W1658008008","https://openalex.org/W1757796397","https://openalex.org/W1870822514","https://openalex.org/W2048226872","https://openalex.org/W2076118331","https://openalex.org/W2107741520","https://openalex.org/W2121863487","https://openalex.org/W2141559645","https://openalex.org/W2145339207","https://openalex.org/W2155968351","https://openalex.org/W2159420891","https://openalex.org/W2173248099","https://openalex.org/W2173564293","https://openalex.org/W2741122588","https://openalex.org/W2766447205","https://openalex.org/W2772721022","https://openalex.org/W2775482448","https://openalex.org/W2785626633","https://openalex.org/W2892979040","https://openalex.org/W2897025488","https://openalex.org/W2905606790","https://openalex.org/W2946901134","https://openalex.org/W2963296584","https://openalex.org/W2963376229","https://openalex.org/W2963477884","https://openalex.org/W2963582482","https://openalex.org/W2964001908","https://openalex.org/W2964291307","https://openalex.org/W2965870268","https://openalex.org/W3148685027","https://openalex.org/W4245108548","https://openalex.org/W6683452725","https://openalex.org/W6684921986","https://openalex.org/W6685444567","https://openalex.org/W6687681856","https://openalex.org/W6730111887","https://openalex.org/W6740801417","https://openalex.org/W6744838376","https://openalex.org/W6748554570","https://openalex.org/W6748613090","https://openalex.org/W6748645729","https://openalex.org/W6780404908","https://openalex.org/W6786431322","https://openalex.org/W6842469858"],"related_works":["https://openalex.org/W3201993717","https://openalex.org/W2526096220","https://openalex.org/W2793798239","https://openalex.org/W3036738089","https://openalex.org/W3107781883","https://openalex.org/W3006416029","https://openalex.org/W2978041660","https://openalex.org/W2909521941","https://openalex.org/W3033477500","https://openalex.org/W2906797721","https://openalex.org/W2787800669","https://openalex.org/W2901569818","https://openalex.org/W2982753101","https://openalex.org/W2782108898","https://openalex.org/W2963959137","https://openalex.org/W2136424116","https://openalex.org/W2912496897","https://openalex.org/W3187600255","https://openalex.org/W3135865540","https://openalex.org/W2954804306"],"abstract_inverted_index":{"Prioritized":[0],"experience":[1],"replay":[2],"(PER)":[3],"samples":[4,69],"important":[5],"transitions,":[6],"rather":[7],"than":[8],"uniformly,":[9],"to":[10,31,78,121],"improve":[11],"data":[12],"efficiency":[13],"of":[14,68],"a":[15],"deep":[16,34],"reinforcement":[17],"learning":[18,81],"agent.":[19],"We":[20],"claim":[21],"that":[22,110],"such":[23],"prioritization":[24],"must":[25],"be":[26],"balanced":[27],"with":[28,106],"sample":[29],"diversity":[30,67],"make":[32],"the":[33,66,80,90,94],"Q-network":[35],"(DQN)":[36],"stabilized":[37],"and":[38,62,70,103,117],"prevent":[39],"severe":[40],"forgetting.":[41],"Our":[42],"proposed":[43],"improvement":[44],"over":[45,126],"PER,":[46],"called":[47],"Predictive":[48],"PER":[49],"(PPER),":[50],"takes":[51],"three":[52,91],"countermeasures":[53],"(TDInit,":[54],"TDClip,":[55],"TDPred)":[56],"for":[57,98],"(i)":[58],"eliminating":[59],"priority":[60],"outliers":[61],"explosions;":[63],"(ii)":[64],"improving":[65],"their":[71],"distributions,":[72],"weighted":[73],"by":[74],"priorities.":[75,101],"Both":[76],"contribute":[77,120],"stabilizing":[79],"process,":[82],"thus":[83],"forgetting":[84],"less.":[85],"The":[86],"most":[87],"notable":[88],"among":[89],"is":[92],"TDPred,":[93],"second":[95],"DNN":[96],"introduced":[97],"generalizing":[99],"in-distribution":[100],"Ablation":[102],"experimental":[104],"studies":[105],"Atari":[107],"games":[108],"show":[109],"each":[111],"countermeasure,":[112],"in":[113],"its":[114],"own":[115],"way,":[116],"PPER":[118],"successfully":[119],"enhancing":[122],"stability":[123],"hence":[124],"performance,":[125],"PER.":[127]},"counts_by_year":[{"year":2021,"cited_by_count":1}],"updated_date":"2026-08-05T07:39:15.569665","created_date":"2025-10-10T00:00:00"}