{"id":"https://openalex.org/W7214219299","doi":"https://doi.org/10.48550/arxiv.2609.26918","title":"On Preference Coverage Collapse from Hindsight Relabeling in Multi-Objective Reinforcement Learning","display_name":"On Preference Coverage Collapse from Hindsight Relabeling in Multi-Objective Reinforcement Learning","publication_year":2026,"publication_date":"2026-09-22","ids":{"openalex":"https://openalex.org/W7214219299","doi":"https://doi.org/10.48550/arxiv.2609.26918"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.26918","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.26918","pdf_url":"https://arxiv.org/pdf/2609.26918","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.26918","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5120305864","display_name":"Baptiste Bonin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I4210164802","display_name":"Mila - Quebec Artificial Intelligence Institute","ror":"https://ror.org/05c22rx21","country_code":"CA","type":"facility","lineage":["https://openalex.org/I4210164802"]},{"id":"https://openalex.org/I43406934","display_name":"Universit\u00e9 Laval","ror":"https://ror.org/04sjchr03","country_code":"CA","type":"education","lineage":["https://openalex.org/I43406934"]}],"countries":["CA"],"is_corresponding":false,"raw_author_name":"Bonin, Baptiste","raw_affiliation_strings":["Mila (Quebec AI Institute)","Universit\u00e9 Laval"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Mila (Quebec AI Institute)","institution_ids":["https://openalex.org/I4210164802"]},{"raw_affiliation_string":"Universit\u00e9 Laval","institution_ids":["https://openalex.org/I43406934"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133979938","display_name":"Caro Strickland","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Strickland, Caro","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5152761481","display_name":"Audrey Durand","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Durand, Audrey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":1.0,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10848","display_name":"Advanced Multi-Objective Optimization Algorithms","score":9.999999747378752e-05,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12794","display_name":"Adaptive Dynamic Programming Control","score":0.0,"subfield":{"id":"https://openalex.org/subfields/2207","display_name":"Control and Systems Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/multi-objective-reinforcement-learning","display_name":"multi-objective reinforcement learning","score":0.9459999799728394},{"id":"https://openalex.org/keywords/sample-efficiency","display_name":"sample efficiency","score":0.6309999823570251},{"id":"https://openalex.org/keywords/continuous-control","display_name":"continuous control","score":0.6959999799728394}],"concepts":[{"id":"https://openalex.org/C10347200","wikidata":"https://www.wikidata.org/wiki/Q1960297","display_name":"Hindsight bias","level":2,"score":0.8159000277519226},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6459000110626221},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.5217000246047974},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.4828000068664551},{"id":"https://openalex.org/C148220186","wikidata":"https://www.wikidata.org/wiki/Q7111912","display_name":"Outcome (game theory)","level":2,"score":0.4779999852180481},{"id":"https://openalex.org/C2778029271","wikidata":"https://www.wikidata.org/wiki/Q5421931","display_name":"Extension (predicate logic)","level":2,"score":0.4404999911785126},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.43369999527931213},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.40540000796318054},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.36399999260902405},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.3573000133037567},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.3497999906539917},{"id":"https://openalex.org/C2777363581","wikidata":"https://www.wikidata.org/wiki/Q15098235","display_name":"Harm","level":2,"score":0.32589998841285706},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.32249999046325684},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.32100000977516174},{"id":"https://openalex.org/C67203356","wikidata":"https://www.wikidata.org/wiki/Q1321905","display_name":"Reinforcement","level":2,"score":0.3174000084400177},{"id":"https://openalex.org/C99498987","wikidata":"https://www.wikidata.org/wiki/Q2210247","display_name":"Noise (video)","level":3,"score":0.31619998812675476},{"id":"https://openalex.org/C204323151","wikidata":"https://www.wikidata.org/wiki/Q905424","display_name":"Range (aeronautics)","level":2,"score":0.3151000142097473},{"id":"https://openalex.org/C144237770","wikidata":"https://www.wikidata.org/wiki/Q747534","display_name":"Mathematical economics","level":1,"score":0.3142000138759613},{"id":"https://openalex.org/C77553402","wikidata":"https://www.wikidata.org/wiki/Q13222579","display_name":"Upper and lower bounds","level":2,"score":0.3125},{"id":"https://openalex.org/C89128539","wikidata":"https://www.wikidata.org/wiki/Q1949963","display_name":"Statistic","level":2,"score":0.3109000027179718},{"id":"https://openalex.org/C2777317252","wikidata":"https://www.wikidata.org/wiki/Q18393516","display_name":"Rare events","level":2,"score":0.3012000024318695},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.2863999903202057},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.2818000018596649},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.26930001378059387},{"id":"https://openalex.org/C162324750","wikidata":"https://www.wikidata.org/wiki/Q8134","display_name":"Economics","level":0,"score":0.25780001282691956}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.26918","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.26918","pdf_url":"https://arxiv.org/pdf/2609.26918","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.26918","doi":"https://doi.org/10.48550/arxiv.2609.26918","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.26918","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.26918","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.26918","pdf_url":"https://arxiv.org/pdf/2609.26918","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214219299.pdf","grobid_xml":"https://content.openalex.org/works/W7214219299.grobid-xml"},"referenced_works_count":9,"referenced_works":["https://openalex.org/W2173248099","https://openalex.org/W2201581102","https://openalex.org/W2530195778","https://openalex.org/W2788741142","https://openalex.org/W4302056481","https://openalex.org/W4317668884","https://openalex.org/W7128490639","https://openalex.org/W7160902816","https://openalex.org/W7202156639"],"related_works":["https://openalex.org/W7202156639","https://openalex.org/W3177105052","https://openalex.org/W4390647558","https://openalex.org/W4407466734","https://openalex.org/W4402443793","https://openalex.org/W3100358394","https://openalex.org/W3129908851","https://openalex.org/W7133488227","https://openalex.org/W7160915748","https://openalex.org/W7157225544"],"abstract_inverted_index":{"Hindsight":[0],"relabeling":[1,124,220,245],"which":[2,219],"retroactively":[3],"replacing":[4],"a":[5,101,158,168,179],"transition's":[6],"goal":[7],"with":[8,36,153],"the":[9,11,37,40,45,71,94,107,126,134,137,163,184,189,205,215,234],"outcome":[10],"agent":[12,41,138],"actually":[13],"achieved":[14,42,185],"is":[15,54,99,241],"an":[16],"effective":[17],"tool":[18],"for":[19,247],"improving":[20],"sample-efficiency":[21],"in":[22,218],"Reinforcement":[23],"Learning":[24],"(RL).":[25],"A":[26],"natural":[27],"extension":[28,53],"to":[29,140,209,229],"preference-conditioned":[30,59],"multi-objective":[31],"RL":[32],"(MORL)":[33],"relabels":[34],"transitions":[35],"preference":[38,135,155,225,235],"direction":[39,186],"rather":[43],"than":[44],"one":[46,193,216],"asked":[47],"for.":[48],"We":[49,142,175],"show":[50],"that":[51,161],"this":[52,144],"frequently":[55],"harmful:":[56],"across":[57,196],"four":[58,86],"off-policy":[60],"algorithms":[61],"spanning":[62],"two":[63,67],"critic":[64],"backbones":[65],"and":[66,92,112,150,199,212,222],"preference-sampling":[68],"schemes":[69],"on":[70],"continuous-control":[72],"MO-Gymnasium":[73],"suite,":[74],"it":[75,152,201],"degrades":[76],"19":[77,206],"of":[78,103,133,204],"36":[79],"algorithm-environment":[80],"settings":[81,208],"by":[82],"as":[83,85],"much":[84],"standard":[87],"deviations,":[88],"improves":[89,214],"only":[90],"one,":[91],"leaves":[93],"rest":[95],"unaffected.":[96],"The":[97],"harm":[98,164],"not":[100,237],"symptom":[102],"noisy":[104,239],"relabels;":[105],"denoising":[106],"target":[108],"recovers":[109],"almost":[110],"nothing,":[111],"neither":[113],"prioritized":[114],"sampling":[115],"nor":[116],"any":[117],"buffer-structural":[118],"choice":[119],"reproduces":[120],"it.":[121],"Instead,":[122],"repeated":[123],"collapses":[125],"critic's":[127],"coverage":[128,171,232],"onto":[129],"whatever":[130],"narrow":[131],"region":[132],"space":[136],"happened":[139],"visit.":[141],"name":[143],"failure":[145],"mode":[146],"\\emph{Preference":[147],"Coverage":[148],"Collapse},":[149],"quantify":[151],"abandoned":[154,224],"mass":[156,226],"(APM),":[157],"value-aware":[159],"statistic":[160],"tracks":[162],"($\u03c1=":[165],"-0.73$)":[166],"where":[167],"purely":[169],"structural":[170],"count":[172],"does":[173],"not.":[174],"then":[176],"introduce":[177],"\\texttt{her\\_mix},":[178],"single-parameter":[180],"convex":[181],"combination":[182],"pulling":[183],"back":[187],"towards":[188],"requested":[190],"preference.":[191],"At":[192],"fixed":[194],"value":[195],"every":[197],"algorithm":[198],"environment,":[200],"returns":[202],"16":[203],"harmed":[207],"baseline,":[210],"preserves":[211],"even":[213],"setting":[217],"helps,":[221],"cuts":[223],"from":[227],"$69\\%$":[228],"$6\\%$.":[230],"Protecting":[231],"over":[233],"simplex,":[236],"filtering":[238],"relabels,":[240],"what":[242],"makes":[243],"hindsight":[244],"safe":[246],"MORL.":[248]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-25T00:00:00"}