{"id":"https://openalex.org/W7166706070","doi":"https://doi.org/10.48550/arxiv.2606.30339","title":"REAR: Test-time Preference Realignment through Reward Decomposition","display_name":"REAR: Test-time Preference Realignment through Reward Decomposition","publication_year":2026,"publication_date":"2026-06-29","ids":{"openalex":"https://openalex.org/W7166706070","doi":"https://doi.org/10.48550/arxiv.2606.30339"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.30339","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30339","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.30339","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5047301622","display_name":"F Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Fuxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139637188","display_name":"Pengcheng Wang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Wang, Pengcheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139692474","display_name":"Chenran Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Chenran","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139676254","display_name":"Yi-Chen Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Yi-Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139645973","display_name":"Yuxin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yuxin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139647961","display_name":"Lang Feng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Feng, Lang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139688538","display_name":"Chenfeng Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Chenfeng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5139638818","display_name":"Masayoshi Tomizuka","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Tomizuka, Masayoshi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139705058","display_name":"Bo An","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"An, Bo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5170000195503235,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11714","display_name":"Multimodal Machine Learning Applications","score":0.5170000195503235,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1526000052690506,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10203","display_name":"Recommender Systems and Techniques","score":0.04259999841451645,"subfield":{"id":"https://openalex.org/subfields/1710","display_name":"Information Systems"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.6705999970436096},{"id":"https://openalex.org/keywords/preference-elicitation","display_name":"Preference elicitation","score":0.6359000205993652},{"id":"https://openalex.org/keywords/scalability","display_name":"Scalability","score":0.5516999959945679},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5030999779701233},{"id":"https://openalex.org/keywords/task","display_name":"Task (project management)","score":0.5001000165939331},{"id":"https://openalex.org/keywords/correctness","display_name":"Correctness","score":0.4936000108718872},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.4325000047683716},{"id":"https://openalex.org/keywords/base","display_name":"Base (topology)","score":0.3698999881744385}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6962000131607056},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6705999970436096},{"id":"https://openalex.org/C2777868144","wikidata":"https://www.wikidata.org/wiki/Q7239817","display_name":"Preference elicitation","level":3,"score":0.6359000205993652},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.5516999959945679},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5030999779701233},{"id":"https://openalex.org/C2780451532","wikidata":"https://www.wikidata.org/wiki/Q759676","display_name":"Task (project management)","level":2,"score":0.5001000165939331},{"id":"https://openalex.org/C55439883","wikidata":"https://www.wikidata.org/wiki/Q360812","display_name":"Correctness","level":2,"score":0.4936000108718872},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.4325000047683716},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42329999804496765},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3781999945640564},{"id":"https://openalex.org/C42058472","wikidata":"https://www.wikidata.org/wiki/Q810214","display_name":"Base (topology)","level":2,"score":0.3698999881744385},{"id":"https://openalex.org/C85847156","wikidata":"https://www.wikidata.org/wiki/Q59015987","display_name":"Verifiable secret sharing","level":3,"score":0.3686999976634979},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.35690000653266907},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3249000012874603},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.3183000087738037},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C2778755073","wikidata":"https://www.wikidata.org/wiki/Q10858537","display_name":"Scale (ratio)","level":2,"score":0.290800005197525},{"id":"https://openalex.org/C94822996","wikidata":"https://www.wikidata.org/wiki/Q1777902","display_name":"Satisficing","level":2,"score":0.2849999964237213},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.2727999985218048},{"id":"https://openalex.org/C124681953","wikidata":"https://www.wikidata.org/wiki/Q339062","display_name":"Decomposition","level":2,"score":0.2721000015735626},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.26570001244544983},{"id":"https://openalex.org/C99844830","wikidata":"https://www.wikidata.org/wiki/Q102441924","display_name":"Scaling","level":2,"score":0.2619999945163727},{"id":"https://openalex.org/C36464697","wikidata":"https://www.wikidata.org/wiki/Q451553","display_name":"Visualization","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C175154964","wikidata":"https://www.wikidata.org/wiki/Q380077","display_name":"Task analysis","level":3,"score":0.2556000053882599}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.30339","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30339","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.30339","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.30339","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Aligning":[0],"large":[1],"language":[2],"models":[3,20,73],"(LLMs)":[4],"with":[5,89,160],"diverse":[6,191],"user":[7,192],"preferences":[8],"is":[9,58,96],"a":[10,69,77,122,145],"critical":[11],"yet":[12],"challenging":[13],"task.":[14],"While":[15],"post-training":[16],"methods":[17],"can":[18,141],"adapt":[19],"to":[21,48,64,86,97,108,114,120,158,175,197],"specific":[22],"needs,":[23],"they":[24],"often":[25,84],"require":[26],"costly":[27],"data":[28],"curation":[29],"and":[30,53,111,156,168,199],"additional":[31],"training.":[32],"Test-time":[33],"scaling":[34],"(TTS)":[35],"presents":[36],"an":[37],"efficient,":[38],"training-free":[39],"alternative,":[40],"but":[41,194],"its":[42],"application":[43],"has":[44],"been":[45],"largely":[46],"limited":[47],"verifiable":[49],"domains":[50],"like":[51],"mathematics":[52],"coding,":[54],"where":[55],"response":[56],"correctness":[57],"easily":[59],"judged.":[60],"To":[61],"extend":[62],"TTS":[63,162],"preference":[65,115,187,204],"alignment,":[66],"we":[67],"introduce":[68],"novel":[70],"framework":[71],"that":[72,126,139,173],"the":[74,81,90,99,109,112,129],"task":[75],"as":[76,144,165],"realignment":[78,185],"problem,":[79],"since":[80],"base":[82],"model":[83],"fails":[85],"sufficiently":[87],"align":[88],"stated":[91],"preference.":[92],"Our":[93],"key":[94],"insight":[95],"decompose":[98],"underlying":[100],"reward":[101,134],"function":[102],"into":[103],"two":[104,133],"components:":[105],"one":[106],"related":[107],"question":[110],"other":[113,176],"information.":[116],"This":[117],"allows":[118],"us":[119],"derive":[121],"REAlignment":[123],"Reward":[124],"(REAR)":[125],"selectively":[127],"rescales":[128],"proportions":[130],"of":[131,148],"these":[132],"terms.":[135],"We":[136],"then":[137],"show":[138,172],"REAR":[140,179],"be":[142],"formulated":[143],"linear":[146],"combination":[147],"token-level":[149],"policy":[150],"log-probabilities,":[151],"making":[152],"it":[153],"computationally":[154],"efficient":[155],"easy":[157],"integrate":[159],"various":[161],"algorithms":[163],"such":[164],"best-of-$N$":[166],"sampling":[167],"tree":[169],"search.":[170],"Experiments":[171],"compared":[174],"test-time":[177,184],"baselines,":[178],"not":[180],"only":[181],"enables":[182],"scalable":[183],"for":[186],"alignment":[188],"tasks":[189,201],"under":[190,202],"requirements,":[193],"also":[195],"generalizes":[196],"mathematical":[198],"visual":[200],"appropriate":[203],"settings.":[205]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-07-01T00:00:00"}