{"id":"https://openalex.org/W4393023366","doi":"https://doi.org/10.48550/arxiv.2403.10946","title":"The Fallacy of Minimizing Cumulative Regret in the Sequential Task Setting","display_name":"The Fallacy of Minimizing Cumulative Regret in the Sequential Task Setting","publication_year":2024,"publication_date":"2024-03-16","ids":{"openalex":"https://openalex.org/W4393023366","doi":"https://doi.org/10.48550/arxiv.2403.10946"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2403.10946","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.10946","pdf_url":"https://arxiv.org/pdf/2403.10946","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2403.10946","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5007965098","display_name":"Ziping Xu","orcid":"https://orcid.org/0000-0002-2591-0356","observed_orcids":["https://orcid.org/0000-0002-2591-0356"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Xu, Ziping","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Zhang, Kelly W.","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I47508984","display_name":"Imperial College London","ror":"https://ror.org/041kmwe10","country_code":"GB","type":"education","lineage":["https://openalex.org/I47508984"]}],"countries":["GB"],"is_corresponding":false,"raw_author_name":"Zhang, Kelly W.","raw_affiliation_strings":["Imperial College London"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Imperial College London","institution_ids":["https://openalex.org/I47508984"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5070589969","display_name":"Susan A. Murphy","orcid":"https://orcid.org/0000-0002-2032-4286","observed_orcids":["https://orcid.org/0000-0002-2032-4286"]},"institutions":[{"id":"https://openalex.org/I136199984","display_name":"Harvard University","ror":"https://ror.org/03vek6s52","country_code":"US","type":"education","lineage":["https://openalex.org/I136199984"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Murphy, Susan A.","raw_affiliation_strings":["Harvard University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Harvard University","institution_ids":["https://openalex.org/I136199984"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},"topics":[{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.9937000274658203,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.006099999882280827,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12072","display_name":"Machine Learning and Algorithms","score":0.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/online-reinforcement-learning","display_name":"online reinforcement learning","score":0.9620000123977661},{"id":"https://openalex.org/keywords/cumulative-regret","display_name":"cumulative regret","score":0.9660000205039978},{"id":"https://openalex.org/keywords/simple-regret","display_name":"simple regret","score":0.9829999804496765},{"id":"https://openalex.org/keywords/sequential-tasks","display_name":"sequential tasks","score":0.640999972820282},{"id":"https://openalex.org/keywords/warm-start","display_name":"warm start","score":0.6520000100135803},{"id":"https://openalex.org/keywords/exploration-exploitation-trade-off","display_name":"exploration-exploitation balance","score":0.8629999756813049},{"id":"https://openalex.org/keywords/regret-bounds","display_name":"regret bounds","score":0.9129999876022339},{"id":"https://openalex.org/keywords/clinical-trials","display_name":"clinical trials","score":0.6470000147819519},{"id":"https://openalex.org/keywords/human-in-the-loop-decision-making","display_name":"human-in-the-loop decision making","score":0.7710000276565552}],"concepts":[{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.9376815557479858},{"id":"https://openalex.org/C2781035248","wikidata":"https://www.wikidata.org/wiki/Q186150","display_name":"Fallacy","level":2,"score":0.7687820196151733},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.7000117301940918},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4630480110645294},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.4133973717689514},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.3626973628997803},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.33741819858551025},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.30437132716178894},{"id":"https://openalex.org/C111472728","wikidata":"https://www.wikidata.org/wiki/Q9471","display_name":"Epistemology","level":1,"score":0.2072296142578125},{"id":"https://openalex.org/C138885662","wikidata":"https://www.wikidata.org/wiki/Q5891","display_name":"Philosophy","level":0,"score":0.1708155870437622},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.11582586169242859},{"id":"https://openalex.org/C187736073","wikidata":"https://www.wikidata.org/wiki/Q2920921","display_name":"Management","level":1,"score":0.07246047258377075}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2403.10946","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.10946","pdf_url":"https://arxiv.org/pdf/2403.10946","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2403.10946","doi":"https://doi.org/10.48550/arxiv.2403.10946","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2403.10946","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2403.10946","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2403.10946","pdf_url":"https://arxiv.org/pdf/2403.10946","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4393023366.pdf","grobid_xml":"https://content.openalex.org/works/W4393023366.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":["https://openalex.org/W2966613056","https://openalex.org/W7133246579","https://openalex.org/W3186628802","https://openalex.org/W3092302772","https://openalex.org/W2771654510","https://openalex.org/W7133187241","https://openalex.org/W7096456820","https://openalex.org/W7133241709","https://openalex.org/W3033436925","https://openalex.org/W3125946732"],"abstract_inverted_index":{"Online":[0],"Reinforcement":[1],"Learning":[2],"(RL)":[3],"is":[4,39,52,64,165],"typically":[5],"framed":[6],"as":[7,183],"the":[8,32,36,43,49,84,87,138,150,175],"process":[9],"of":[10,29,48,83,86,154,177],"minimizing":[11,59],"cumulative":[12],"regret":[13,56],"(CR)":[14],"through":[15],"interactions":[16],"with":[17],"an":[18],"unknown":[19],"environment.":[20],"However,":[21],"real-world":[22,94],"RL":[23,178],"applications":[24],"usually":[25],"involve":[26],"a":[27,66,125,145],"sequence":[28],"tasks,":[30],"and":[31,62,131,185],"data":[33],"collected":[34],"in":[35,74,81,93,102,106,167,180],"first":[37],"task":[38,121,172],"used":[40],"to":[41,124,144,170],"warm-start":[42,50],"second":[44],"task.":[45],"The":[46],"performance":[47],"policy":[51],"measured":[53],"by":[54],"simple":[55],"(SR).":[57],"While":[58],"both":[60,77],"CR":[61,130,146],"SR":[63],"generally":[65],"conflicting":[67],"objective,":[68],"previous":[69],"research":[70],"has":[71],"shown":[72],"that":[73,120,162],"stationary":[75],"environments,":[76],"can":[78],"be":[79],"optimized":[80],"terms":[82],"duration":[85],"task,":[88],"$T$.":[89],"In":[90],"practice,":[91],"however,":[92],"applications,":[95],"human-in-the-loop":[96],"decisions":[97],"between":[98,115,129],"tasks":[99],"often":[100],"results":[101,118],"non-stationarity.":[103],"For":[104],"instance,":[105],"clinical":[107],"trials,":[108],"scientists":[109],"may":[110],"adjust":[111],"target":[112],"health":[113],"outcomes":[114],"implementations.":[116],"Our":[117],"show":[119],"non-stationarity":[122],"leads":[123],"more":[126],"restrictive":[127],"trade-off":[128],"SR.":[132],"To":[133],"balance":[134],"these":[135],"competing":[136],"goals,":[137],"algorithm":[139],"must":[140],"explore":[141],"excessively,":[142],"leading":[143],"bound":[147],"worse":[148],"than":[149],"typical":[151],"optimal":[152],"rate":[153],"$T^{1/2}$.":[155],"These":[156],"findings":[157],"are":[158],"practically":[159],"significant,":[160],"indicating":[161],"increased":[163],"exploration":[164],"necessary":[166],"non-stationary":[168],"environments":[169],"accommodate":[171],"changes,":[173],"impacting":[174],"design":[176],"algorithms":[179],"fields":[181],"such":[182],"healthcare":[184],"beyond.":[186]},"counts_by_year":[],"updated_date":"2026-10-14T16:39:01.820549","created_date":"2024-03-21T00:00:00"}