{"id":"https://openalex.org/W7162409522","doi":"https://doi.org/10.48550/arxiv.2605.25629","title":"When In-Distribution Gains Fail: Evaluating Weak-to-Strong Reward Models under Preference Shift","display_name":"When In-Distribution Gains Fail: Evaluating Weak-to-Strong Reward Models under Preference Shift","publication_year":2026,"publication_date":"2026-05-25","ids":{"openalex":"https://openalex.org/W7162409522","doi":"https://doi.org/10.48550/arxiv.2605.25629"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.25629","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.25629","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5135944908","display_name":"Khoi Le","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Le, Khoi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135929628","display_name":"Tri Cao","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Cao, Tri","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137001477","display_name":"Phong Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Phong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137028944","display_name":"Cong-Duy Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Cong-Duy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137037797","display_name":"Anh Tuan Luu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Luu, Anh Tuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137057255","display_name":"Miao Chunyan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chunyan, Miao","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137033625","display_name":"See-Kiong Ng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ng, See-Kiong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137057127","display_name":"Thong Van Nguyen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Nguyen, Thong","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.3127000033855438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.3127000033855438,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.1800999939441681,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.0828000009059906,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.762499988079071},{"id":"https://openalex.org/keywords/generalization","display_name":"Generalization","score":0.6689000129699707},{"id":"https://openalex.org/keywords/representation","display_name":"Representation (politics)","score":0.6018999814987183},{"id":"https://openalex.org/keywords/preference-learning","display_name":"Preference learning","score":0.541100025177002},{"id":"https://openalex.org/keywords/anchoring","display_name":"Anchoring","score":0.5260000228881836},{"id":"https://openalex.org/keywords/space","display_name":"Space (punctuation)","score":0.46549999713897705},{"id":"https://openalex.org/keywords/path","display_name":"Path (computing)","score":0.34709998965263367}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.762499988079071},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.6689000129699707},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.6018999814987183},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.541100025177002},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.52920001745224},{"id":"https://openalex.org/C18483071","wikidata":"https://www.wikidata.org/wiki/Q168432","display_name":"Anchoring","level":2,"score":0.5260000228881836},{"id":"https://openalex.org/C2778572836","wikidata":"https://www.wikidata.org/wiki/Q380933","display_name":"Space (punctuation)","level":2,"score":0.46549999713897705},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.45899999141693115},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.438400000333786},{"id":"https://openalex.org/C2777735758","wikidata":"https://www.wikidata.org/wiki/Q817765","display_name":"Path (computing)","level":2,"score":0.34709998965263367},{"id":"https://openalex.org/C48044578","wikidata":"https://www.wikidata.org/wiki/Q727490","display_name":"Scalability","level":2,"score":0.32269999384880066},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.3199999928474426},{"id":"https://openalex.org/C149782125","wikidata":"https://www.wikidata.org/wiki/Q160039","display_name":"Econometrics","level":1,"score":0.30230000615119934},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.2994000017642975},{"id":"https://openalex.org/C180747234","wikidata":"https://www.wikidata.org/wiki/Q23373","display_name":"Cognitive psychology","level":1,"score":0.2985000014305115},{"id":"https://openalex.org/C2776436953","wikidata":"https://www.wikidata.org/wiki/Q5163215","display_name":"Consistency (knowledge bases)","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C2781043087","wikidata":"https://www.wikidata.org/wiki/Q939761","display_name":"Preference theory","level":3,"score":0.2703999876976013},{"id":"https://openalex.org/C150899416","wikidata":"https://www.wikidata.org/wiki/Q1820378","display_name":"Transfer of learning","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.2667999863624573},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.26249998807907104},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.2500999867916107}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.25629","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.25629","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.25629","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Weak-to-strong":[0],"(W2S)":[1],"generalization":[2],"is":[3],"a":[4,55,86,144],"promising":[5],"framework":[6],"for":[7,54],"scalable":[8],"oversight,":[9],"yet":[10,88],"existing":[11],"evaluations":[12],"often":[13],"test":[14],"students":[15,34],"under":[16,26],"matched":[17],"train-test":[18],"distributions.":[19],"Therefore,":[20],"we":[21,81],"study":[22],"W2S":[23,139],"preference":[24,38,49,76,110,150],"learning":[25],"zero-shot":[27],"distribution":[28],"shift":[29],"and":[30,113,132,142],"find":[31],"that":[32,91],"strong":[33,66,98],"trained":[35],"on":[36],"weak":[37],"labels":[39],"can":[40,63],"appear":[41],"successful":[42],"in-distribution":[43,124],"while":[44,104,121],"failing":[45],"to":[46],"transfer":[47,120],"across":[48],"datasets.":[50],"We":[51],"provide":[52,143],"evidence":[53],"representational":[56],"failure":[57],"mode":[58],"in":[59,137],"which":[60],"weak-supervised":[61],"fine-tuning":[62],"pull":[64],"the":[65,96],"model":[67,114],"toward":[68,147],"source-domain":[69],"features":[70],"instead":[71],"of":[72],"maintaining":[73,122],"broadly":[74],"transferable":[75],"representations.":[77],"To":[78],"mitigate":[79],"this,":[80],"propose":[82],"Representation":[83],"Anchoring":[84],"(Anchor),":[85],"simple":[87],"effective":[89],"regularizer":[90],"constrains":[92],"excessive":[93],"drift":[94],"from":[95],"pretrained":[97],"model's":[99],"representation":[100],"space":[101],"during":[102],"fine-tuning,":[103],"still":[105],"allowing":[106],"task-relevant":[107],"adaptation.":[108],"Across":[109],"domains,":[111],"datasets,":[112],"families,":[115],"Anchor":[116],"consistently":[117],"improves":[118],"out-of-distribution":[119],"competitive":[123],"performance.":[125],"Together,":[126],"our":[127],"evaluation":[128],"protocol,":[129],"transfer-aware":[130],"metrics,":[131],"method":[133],"expose":[134],"hidden":[135],"brittleness":[136],"current":[138],"reward":[140],"modeling":[141],"practical":[145],"path":[146],"more":[148],"robust":[149],"transfer.":[151]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-27T00:00:00"}