{"id":"https://openalex.org/W3132227397","doi":"https://doi.org/10.48550/arxiv.2102.11075","title":"SENTINEL: Taming Uncertainty with Ensemble-based Distributional Reinforcement Learning","display_name":"SENTINEL: Taming Uncertainty with Ensemble-based Distributional Reinforcement Learning","publication_year":2021,"publication_date":"2021-02-22","ids":{"openalex":"https://openalex.org/W3132227397","doi":"https://doi.org/10.48550/arxiv.2102.11075","mag":"3132227397"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2102.11075","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.11075","pdf_url":"https://arxiv.org/pdf/2102.11075","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2102.11075","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5028876789","display_name":"Hannes Eriksson","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I66862912","display_name":"Chalmers University of Technology","ror":"https://ror.org/040wg7k59","country_code":"SE","type":"education","lineage":["https://openalex.org/I66862912"]}],"countries":["SE"],"is_corresponding":false,"raw_author_name":"Eriksson, Hannes","raw_affiliation_strings":["Chalmers University of Technology , Gothenburg , Sweden","Zenseact AB , Gothenburg , Sweden","Zenseact AB"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chalmers University of Technology , Gothenburg , Sweden","institution_ids":["https://openalex.org/I66862912"]},{"raw_affiliation_string":"Zenseact AB , Gothenburg , Sweden","institution_ids":[]},{"raw_affiliation_string":"Zenseact AB","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5058797341","display_name":"Debabrota Basu","orcid":"https://orcid.org/0000-0002-3204-2884","observed_orcids":["https://orcid.org/0000-0002-3204-2884"]},"institutions":[{"id":"https://openalex.org/I1294671590","display_name":"Centre National de la Recherche Scientifique","ror":"https://ror.org/02feahw73","country_code":"FR","type":"government","lineage":["https://openalex.org/I1294671590"]},{"id":"https://openalex.org/I1326498283","display_name":"Institut national de recherche en sciences et technologies du num\u00e9rique","ror":"https://ror.org/02kvxyf05","country_code":"FR","type":"government","lineage":["https://openalex.org/I1326498283"]},{"id":"https://openalex.org/I4210138412","display_name":"Centre Inria de l'Universit\u00e9 de Lille","ror":"https://ror.org/04eej9726","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1326498283","https://openalex.org/I4210138412"]},{"id":"https://openalex.org/I4387153239","display_name":"Centre de Recherche en Informatique, Signal et Automatique de Lille","ror":"https://ror.org/05vrs3189","country_code":"FR","type":"facility","lineage":["https://openalex.org/I1294671590","https://openalex.org/I1326498283","https://openalex.org/I2279609970","https://openalex.org/I4387153239","https://openalex.org/I7454413"]}],"countries":["FR"],"is_corresponding":false,"raw_author_name":"Basu, Debabrota","raw_affiliation_strings":["CRIStAL , CNRS , Lille , France","Scool , INRIA Lille-Nord Europe , Lille , France","Scool"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"CRIStAL , CNRS , Lille , France","institution_ids":["https://openalex.org/I1294671590","https://openalex.org/I4387153239"]},{"raw_affiliation_string":"Scool , INRIA Lille-Nord Europe , Lille , France","institution_ids":["https://openalex.org/I1326498283","https://openalex.org/I4210138412"]},{"raw_affiliation_string":"Scool","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5083753701","display_name":"Mina Alibeigi","orcid":"https://orcid.org/0000-0002-5078-0194","observed_orcids":["https://orcid.org/0000-0002-5078-0194"]},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]}],"countries":["NO"],"is_corresponding":false,"raw_author_name":"Alibeigi, Mina","raw_affiliation_strings":["University of Oslo , Norway","Zenseact AB , Gothenburg , Sweden","Zenseact AB"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Oslo , Norway","institution_ids":["https://openalex.org/I184942183"]},{"raw_affiliation_string":"Zenseact AB , Gothenburg , Sweden","institution_ids":[]},{"raw_affiliation_string":"Zenseact AB","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5017471254","display_name":"Christos Dimitrakakis A","orcid":"https://orcid.org/0000-0002-5367-5189","observed_orcids":["https://orcid.org/0000-0002-5367-5189"]},"institutions":[{"id":"https://openalex.org/I184942183","display_name":"University of Oslo","ror":"https://ror.org/01xtthb56","country_code":"NO","type":"education","lineage":["https://openalex.org/I184942183"]},{"id":"https://openalex.org/I57825437","display_name":"University of Neuch\u00e2tel","ror":"https://ror.org/00vasag41","country_code":"CH","type":"education","lineage":["https://openalex.org/I57825437"]},{"id":"https://openalex.org/I66862912","display_name":"Chalmers University of Technology","ror":"https://ror.org/040wg7k59","country_code":"SE","type":"education","lineage":["https://openalex.org/I66862912"]}],"countries":["CH","NO","SE"],"is_corresponding":false,"raw_author_name":"Dimitrakakis, Christos","raw_affiliation_strings":["Chalmers University of Technology , Gothenburg , Sweden","University of Neuchatel , Switzerland and","University of Oslo"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Chalmers University of Technology , Gothenburg , Sweden","institution_ids":["https://openalex.org/I66862912"]},{"raw_affiliation_string":"University of Neuchatel , Switzerland and","institution_ids":["https://openalex.org/I57825437"]},{"raw_affiliation_string":"University of Oslo","institution_ids":["https://openalex.org/I184942183"]}]}],"institutions":[],"countries_distinct_count":4,"institutions_distinct_count":7,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":7,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":"180","issue":null,"first_page":"631","last_page":"640"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.9994999766349792,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/aleatoric-uncertainty","display_name":"aleatoric uncertainty","score":0.8320000171661377},{"id":"https://openalex.org/keywords/epistemic-uncertainty","display_name":"epistemic uncertainty","score":0.9980000257492065},{"id":"https://openalex.org/keywords/distributional-reinforcement-learning","display_name":"distributional reinforcement learning","score":0.9129999876022339}],"concepts":[{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.7860174179077148},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6733173131942749},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5215972661972046},{"id":"https://openalex.org/C45942800","wikidata":"https://www.wikidata.org/wiki/Q245652","display_name":"Ensemble learning","level":2,"score":0.42580246925354004},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3764379918575287}],"mesh":[],"locations_count":8,"locations":[{"id":"pmh:oai:arXiv.org:2102.11075","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.11075","pdf_url":"https://arxiv.org/pdf/2102.11075","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:research.chalmers.se:523090","doi":null,"is_oa":true,"landing_page_url":"https://research.chalmers.se/en/publication/523090","pdf_url":"https://research.chalmers.se/en/publication/523090","source":{"id":"https://openalex.org/S4306402469","display_name":"Chalmers Research (Chalmers University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I66862912","host_organization_name":"Chalmers University of Technology","host_organization_lineage":["https://openalex.org/I66862912"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":null},{"id":"pmh:oai:research.chalmers.se:554156","doi":null,"is_oa":true,"landing_page_url":"https://research.chalmers.se/en/publication/9c1c2a2a-7860-4ea2-96e2-c25bb4785c8e","pdf_url":"https://research.chalmers.se/publication/554156/file/554156_Fulltext.pdf","source":{"id":"https://openalex.org/S4306402469","display_name":"Chalmers Research (Chalmers University of Technology)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I66862912","host_organization_name":"Chalmers University of Technology","host_organization_lineage":["https://openalex.org/I66862912"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":null},{"id":"pmh:oai:HAL:hal-03150823v1","doi":null,"is_oa":true,"landing_page_url":"https://hal.archives-ouvertes.fr/hal-03150823","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Proceedings of the Thirty-Eighth Conference on Uncertainty in Artificial Intelligence","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:HAL:hal-03150823v2","doi":null,"is_oa":true,"landing_page_url":"https://hal.science/hal-03150823","pdf_url":null,"source":{"id":"https://openalex.org/S4306402512","display_name":"HAL","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4405252976","host_organization_name":"Centre pour la Communication Scientifique Directe","host_organization_lineage":["https://openalex.org/I4405252976"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by-nc","license_id":"https://openalex.org/licenses/cc-by-nc","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"UAI 2022- Proceedings of the Thirty-Eighth Conference on Uncertainty in Artificial Intelligence, Aug 2022, Eindhoven, Netherlands. pp.631-640","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"pmh:oai:lilloa.univ-lille.fr:20.500.12210/57058","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.12210/57058","pdf_url":null,"source":{"id":"https://openalex.org/S4306402203","display_name":"LillOA (Universit\u00e9 de Lille (University Of Lille))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210123514","host_organization_name":"Centre d'\u00c9tudes en Civilisations, Langues et Lettres \u00c9trang\u00e8res","host_organization_lineage":["https://openalex.org/I4210123514"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/workingPaper"},{"id":"pmh:oai:lilloa.univ-lille.fr:20.500.12210/76721","doi":null,"is_oa":true,"landing_page_url":"http://hdl.handle.net/20.500.12210/76721","pdf_url":null,"source":{"id":"https://openalex.org/S4306402203","display_name":"LillOA (Universit\u00e9 de Lille (University Of Lille))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I4210123514","host_organization_name":"Centre d'\u00c9tudes en Civilisations, Langues et Lettres \u00c9trang\u00e8res","host_organization_lineage":["https://openalex.org/I4210123514"],"host_organization_lineage_names":[],"type":"repository"},"license":"other-oa","license_id":"https://openalex.org/licenses/other-oa","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"","raw_type":"info:eu-repo/semantics/conferenceObject"},{"id":"doi:10.48550/arxiv.2102.11075","doi":"https://doi.org/10.48550/arxiv.2102.11075","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2102.11075","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2102.11075","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2102.11075","pdf_url":"https://arxiv.org/pdf/2102.11075","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G1441096865","display_name":null,"funder_award_id":"2018-05973","funder_id":"https://openalex.org/F4320322327","funder_display_name":"Knut och Alice Wallenbergs Stiftelse"},{"id":"https://openalex.org/G3321450376","display_name":null,"funder_award_id":"05973","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"},{"id":"https://openalex.org/G712881263","display_name":null,"funder_award_id":"2018-","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"},{"id":"https://openalex.org/G796431723","display_name":null,"funder_award_id":"201805973","funder_id":"https://openalex.org/F4320322581","funder_display_name":"Vetenskapsr\u00e5det"}],"funders":[{"id":"https://openalex.org/F4320322327","display_name":"Knut och Alice Wallenbergs Stiftelse","ror":"https://ror.org/004hzzk67"},{"id":"https://openalex.org/F4320322581","display_name":"Vetenskapsr\u00e5det","ror":"https://ror.org/03zttf063"}],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W3132227397.pdf","grobid_xml":"https://content.openalex.org/works/W3132227397.grobid-xml"},"referenced_works_count":38,"referenced_works":["https://openalex.org/W134786152","https://openalex.org/W1570963478","https://openalex.org/W1585575029","https://openalex.org/W1647779468","https://openalex.org/W1845972764","https://openalex.org/W1994372258","https://openalex.org/W2019025561","https://openalex.org/W2019291268","https://openalex.org/W2031355801","https://openalex.org/W2043806097","https://openalex.org/W2057218189","https://openalex.org/W2064556564","https://openalex.org/W2069045459","https://openalex.org/W2084063999","https://openalex.org/W2088413745","https://openalex.org/W2106378743","https://openalex.org/W2111959010","https://openalex.org/W2121863487","https://openalex.org/W2145339207","https://openalex.org/W2149497424","https://openalex.org/W2159223630","https://openalex.org/W2167696704","https://openalex.org/W2280163991","https://openalex.org/W2602963933","https://openalex.org/W2788755844","https://openalex.org/W2886031349","https://openalex.org/W2890803796","https://openalex.org/W2896408693","https://openalex.org/W2917022709","https://openalex.org/W2945850646","https://openalex.org/W2950718433","https://openalex.org/W2962718166","https://openalex.org/W2962977206","https://openalex.org/W2963082979","https://openalex.org/W2963423916","https://openalex.org/W2963590277","https://openalex.org/W2963757175","https://openalex.org/W3035880215"],"related_works":["https://openalex.org/W4382981485","https://openalex.org/W4372183490","https://openalex.org/W4392340324","https://openalex.org/W4378190882","https://openalex.org/W4383469185","https://openalex.org/W4294690570","https://openalex.org/W4307411475","https://openalex.org/W2950718433","https://openalex.org/W7133184701","https://openalex.org/W3160598148"],"abstract_inverted_index":{"In":[0],"this":[1],"paper,":[2],"we":[3,17],"consider":[4],"risk-sensitive":[5,159,163],"sequential":[6],"decision-making":[7],"in":[8],"Reinforcement":[9],"Learning":[10],"(RL).":[11],"Our":[12],"contributions":[13],"are":[14],"two-fold.":[15],"First,":[16],"introduce":[18],"a":[19,64],"novel":[20],"and":[21,36,90,96,109,115,135,150,164],"coherent":[22],"quantification":[23],"of":[24,34,67,121],"risk,":[25,28],"namely":[26],"composite":[27,69,82,138,154],"which":[29],"quantifies":[30],"the":[31,40,60,68,72,81,94,132,137,146],"joint":[32],"effect":[33],"aleatory":[35,47,89,116],"epistemic":[37,49,73,91,114],"risk":[38,50,70,74,83,155],"during":[39],"learning":[41],"process.":[42],"Existing":[43],"works":[44],"considered":[45],"either":[46],"or":[48,52],"individually,":[51],"as":[53],"an":[54,102],"additive":[55,61,97],"combination.":[56],"We":[57,99,140],"prove":[58],"that":[59,143],"formulation":[62],"is":[63,76,84],"particular":[65],"case":[66],"when":[71],"measure":[75],"replaced":[77],"with":[78,153],"expectation.":[79],"Thus,":[80],"more":[85],"sensitive":[86],"to":[87,130],"both":[88],"uncertainty":[92,117],"than":[93,161],"individual":[95],"formulations.":[98],"also":[100],"propose":[101],"algorithm,":[103],"SENTINEL-K,":[104],"based":[105],"on":[106],"ensemble":[107,120],"bootstrapping":[108],"distributional":[110,165],"RL":[111,166],"for":[112],"representing":[113],"respectively.":[118],"The":[119,126],"K":[122],"learners":[123],"uses":[124],"Follow":[125],"Regularised":[127],"Leader":[128],"(FTRL)":[129],"aggregate":[131],"return":[133,147],"distributions":[134],"obtain":[136],"risk.":[139],"experimentally":[141],"verify":[142],"SENTINEL-K":[144],"estimates":[145],"distribution":[148],"better,":[149],"while":[151],"used":[152],"estimates,":[156],"demonstrates":[157],"higher":[158],"performance":[160],"state-of-the-art":[162],"algorithms.":[167]},"counts_by_year":[{"year":2025,"cited_by_count":1},{"year":2023,"cited_by_count":4},{"year":2022,"cited_by_count":1},{"year":2021,"cited_by_count":1}],"updated_date":"2026-11-04T10:54:05.112534","created_date":"2021-03-01T00:00:00"}