{"id":"https://openalex.org/W7165157402","doi":"https://doi.org/10.48550/arxiv.2606.19328","title":"UBP2: Uncertainty-Balanced Preference Planning for Efficient Preference-based Reinforcement Learning","display_name":"UBP2: Uncertainty-Balanced Preference Planning for Efficient Preference-based Reinforcement Learning","publication_year":2026,"publication_date":"2026-06-17","ids":{"openalex":"https://openalex.org/W7165157402","doi":"https://doi.org/10.48550/arxiv.2606.19328"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.19328","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.19328","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.19328","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5057979603","display_name":"Mohamed Nabail","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nabail, Mohamed","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138936937","display_name":"Leo Cheng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cheng, Leo Kaixuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5101502442","display_name":"Jingmin Wang","orcid":"https://orcid.org/0000-0001-5157-7262"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Jingmin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5077198404","display_name":"Nicholas Rhinehart","orcid":"https://orcid.org/0000-0003-4242-1236"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Rhinehart, Nicholas","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6887999773025513,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10462","display_name":"Reinforcement Learning in Robotics","score":0.6887999773025513,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.02889999933540821,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12101","display_name":"Advanced Bandit Algorithms Research","score":0.027799999341368675,"subfield":{"id":"https://openalex.org/subfields/1803","display_name":"Management Science and Operations Research"},"field":{"id":"https://openalex.org/fields/18","display_name":"Decision Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/regret","display_name":"Regret","score":0.7876999974250793},{"id":"https://openalex.org/keywords/pairwise-comparison","display_name":"Pairwise comparison","score":0.7296000123023987},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.6560999751091003},{"id":"https://openalex.org/keywords/sample","display_name":"Sample (material)","score":0.5083000063896179},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.5045999884605408},{"id":"https://openalex.org/keywords/benchmark","display_name":"Benchmark (surveying)","score":0.4973999857902527},{"id":"https://openalex.org/keywords/thompson-sampling","display_name":"Thompson sampling","score":0.4961000084877014},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4611000120639801},{"id":"https://openalex.org/keywords/sublinear-function","display_name":"Sublinear function","score":0.4016999900341034}],"concepts":[{"id":"https://openalex.org/C50817715","wikidata":"https://www.wikidata.org/wiki/Q79895177","display_name":"Regret","level":2,"score":0.7876999974250793},{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7296000123023987},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.6560999751091003},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6090999841690063},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5741000175476074},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5315999984741211},{"id":"https://openalex.org/C198531522","wikidata":"https://www.wikidata.org/wiki/Q485146","display_name":"Sample (material)","level":2,"score":0.5083000063896179},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.5045999884605408},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.4973999857902527},{"id":"https://openalex.org/C73602740","wikidata":"https://www.wikidata.org/wiki/Q7795822","display_name":"Thompson sampling","level":3,"score":0.4961000084877014},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4611000120639801},{"id":"https://openalex.org/C117160843","wikidata":"https://www.wikidata.org/wiki/Q338652","display_name":"Sublinear function","level":2,"score":0.4016999900341034},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.39480000734329224},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.3490999937057495},{"id":"https://openalex.org/C2776291640","wikidata":"https://www.wikidata.org/wiki/Q2912517","display_name":"Value (mathematics)","level":2,"score":0.3490999937057495},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.336899995803833},{"id":"https://openalex.org/C14646407","wikidata":"https://www.wikidata.org/wiki/Q1430750","display_name":"Bellman equation","level":2,"score":0.32359999418258667},{"id":"https://openalex.org/C133462117","wikidata":"https://www.wikidata.org/wiki/Q4929239","display_name":"Data collection","level":2,"score":0.28619998693466187},{"id":"https://openalex.org/C774472","wikidata":"https://www.wikidata.org/wiki/Q6760393","display_name":"Margin (machine learning)","level":2,"score":0.27720001339912415},{"id":"https://openalex.org/C91873725","wikidata":"https://www.wikidata.org/wiki/Q3445816","display_name":"Function approximation","level":3,"score":0.2759000062942505},{"id":"https://openalex.org/C131806220","wikidata":"https://www.wikidata.org/wiki/Q852705","display_name":"Evaluation function","level":2,"score":0.2750000059604645},{"id":"https://openalex.org/C92424840","wikidata":"https://www.wikidata.org/wiki/Q7912772","display_name":"Value of information","level":2,"score":0.27489998936653137},{"id":"https://openalex.org/C129848803","wikidata":"https://www.wikidata.org/wiki/Q2564360","display_name":"Sample size determination","level":2,"score":0.2632000148296356},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.2612999975681305},{"id":"https://openalex.org/C77967617","wikidata":"https://www.wikidata.org/wiki/Q4677561","display_name":"Active learning (machine learning)","level":2,"score":0.260699987411499}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.19328","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.19328","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.19328","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.19328","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Preference-based":[0],"RL":[1],"provides":[2],"an":[3,102],"approach":[4,47],"to":[5,79,84],"learning":[6],"reward":[7,19],"models":[8,78],"from":[9,32],"pairwise":[10],"comparisons":[11],"of":[12,41,72],"behaviors,":[13],"bypassing":[14],"the":[15,38,58,134],"need":[16],"for":[17,125],"explicit":[18,103],"design.":[20],"However,":[21],"existing":[22],"methods":[23,147],"typically":[24],"rely":[25],"on":[26,133],"passive":[27],"data":[28],"collection":[29],"and":[30,61,75,94,107,128,148],"suffer":[31],"poor":[33],"sample":[34,142],"efficiency,":[35],"especially":[36],"during":[37],"early":[39],"stages":[40],"learning.":[42],"We":[43],"introduce":[44],"a":[45,85],"model-based":[46,150],"that":[48,88],"actively":[49],"directs":[50],"exploration":[51,114],"by":[52],"jointly":[53],"reasoning":[54],"over":[55],"uncertainties":[56],"in":[57],"reward,":[59,73,91],"dynamics,":[60,74],"value":[62,76],"functions.":[63],"Our":[64],"method,":[65],"Uncertainty-Balanced":[66],"Preference":[67],"Planning":[68,97],"(UBP2),":[69],"uses":[70],"ensembles":[71],"function":[77],"evaluate":[80],"candidate":[81],"trajectories":[82],"according":[83],"unified":[86],"score":[87],"combines":[89],"expected":[90],"terminal":[92],"value,":[93],"epistemic":[95],"uncertainty.":[96],"under":[98],"this":[99],"objective":[100],"yields":[101],"tradeoff":[104],"between":[105],"exploitation":[106],"information":[108],"acquisition":[109],"without":[110],"requiring":[111],"ad":[112],"hoc":[113],"heuristics.":[115],"Under":[116],"standard":[117],"regularity":[118],"assumptions,":[119],"we":[120],"establish":[121],"sublinear":[122],"regret":[123],"guarantees":[124],"both":[126],"finite-horizon":[127],"infinite-horizon":[129],"settings.":[130],"Empirically,":[131],"experiments":[132],"Meta-World":[135],"benchmark":[136],"show":[137],"UBP2":[138],"achieves":[139],"substantially":[140],"higher":[141],"efficiency":[143],"than":[144],"model-free":[145],"preference-based":[146],"non-optimistic":[149],"baselines.":[151]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-19T00:00:00"}