{"id":"https://openalex.org/W7163139150","doi":"https://doi.org/10.48550/arxiv.2606.00357","title":"From \"Weak\" Signals to Strong Models: Preference Delta Aggregation with LoRA Merging","display_name":"From \"Weak\" Signals to Strong Models: Preference Delta Aggregation with LoRA Merging","publication_year":2026,"publication_date":"2026-05-29","ids":{"openalex":"https://openalex.org/W7163139150","doi":"https://doi.org/10.48550/arxiv.2606.00357"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.00357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.00357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.00357","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5137668567","display_name":"Qi Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Qi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137678575","display_name":"Siyue Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Siyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137674507","display_name":"Yulin Chen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Chen, Yulin","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5001208205","display_name":"Yuxiang Xue","orcid":"https://orcid.org/0009-0000-7904-429X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xue, Yuxiang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5137672293","display_name":"Ru Peng","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peng, Ru","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5137642212","display_name":"Chen Zhao (225070)","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhao, Chen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.19519999623298645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.19519999623298645,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1200999990105629,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.0997999981045723,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.6136999726295471},{"id":"https://openalex.org/keywords/linear-subspace","display_name":"Linear subspace","score":0.48809999227523804},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.47380000352859497},{"id":"https://openalex.org/keywords/quality","display_name":"Quality (philosophy)","score":0.4697999954223633},{"id":"https://openalex.org/keywords/baseline","display_name":"Baseline (sea)","score":0.32659998536109924},{"id":"https://openalex.org/keywords/data-quality","display_name":"Data quality","score":0.3212999999523163},{"id":"https://openalex.org/keywords/adapter","display_name":"Adapter (computing)","score":0.31859999895095825}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.6818000078201294},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6136999726295471},{"id":"https://openalex.org/C12362212","wikidata":"https://www.wikidata.org/wiki/Q728435","display_name":"Linear subspace","level":2,"score":0.48809999227523804},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.47380000352859497},{"id":"https://openalex.org/C2779530757","wikidata":"https://www.wikidata.org/wiki/Q1207505","display_name":"Quality (philosophy)","level":2,"score":0.4697999954223633},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.4259999990463257},{"id":"https://openalex.org/C124101348","wikidata":"https://www.wikidata.org/wiki/Q172491","display_name":"Data mining","level":1,"score":0.39910000562667847},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.38280001282691956},{"id":"https://openalex.org/C12725497","wikidata":"https://www.wikidata.org/wiki/Q810247","display_name":"Baseline (sea)","level":2,"score":0.32659998536109924},{"id":"https://openalex.org/C24756922","wikidata":"https://www.wikidata.org/wiki/Q1757694","display_name":"Data quality","level":3,"score":0.3212999999523163},{"id":"https://openalex.org/C177284502","wikidata":"https://www.wikidata.org/wiki/Q1005390","display_name":"Adapter (computing)","level":2,"score":0.31859999895095825},{"id":"https://openalex.org/C61797465","wikidata":"https://www.wikidata.org/wiki/Q1188986","display_name":"Term (time)","level":2,"score":0.29820001125335693},{"id":"https://openalex.org/C2779843651","wikidata":"https://www.wikidata.org/wiki/Q7390335","display_name":"SIGNAL (programming language)","level":2,"score":0.29660001397132874},{"id":"https://openalex.org/C82578977","wikidata":"https://www.wikidata.org/wiki/Q16773055","display_name":"Data aggregator","level":3,"score":0.2759000062942505},{"id":"https://openalex.org/C2779110102","wikidata":"https://www.wikidata.org/wiki/Q1323737","display_name":"Revealed preference","level":2,"score":0.26919999718666077},{"id":"https://openalex.org/C113174947","wikidata":"https://www.wikidata.org/wiki/Q2859736","display_name":"Tree (set theory)","level":2,"score":0.2687000036239624},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.26570001244544983},{"id":"https://openalex.org/C181204326","wikidata":"https://www.wikidata.org/wiki/Q7239820","display_name":"Preference learning","level":3,"score":0.26330000162124634},{"id":"https://openalex.org/C32172795","wikidata":"https://www.wikidata.org/wiki/Q4692266","display_name":"Aggregation problem","level":2,"score":0.2565999925136566},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2549000084400177}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.00357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.00357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.00357","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.00357","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"score":0.4584026038646698,"display_name":"Quality Education","id":"https://metadata.un.org/sdg/4"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Training":[0],"strong":[1,67,177],"large":[2],"language":[3],"models":[4,68],"(LLMs)":[5],"requires":[6],"high-quality":[7],"supervision,":[8],"which":[9,46],"is":[10],"often":[11],"scarce.":[12],"Recent":[13],"work":[14],"shows":[15],"that":[16,84,130,152],"paired":[17],"preference":[18,87,102,225],"data":[19],"from":[20,89],"weak-weaker":[21,91],"model":[22,92,178],"pairs":[23],"(e.g.,":[24,69],"Qwen3":[25,70],"4B":[26],"over":[27],"1.7B),":[28],"despite":[29],"the":[30,81,106,176,201,216],"limited":[31],"quality":[32,44],"of":[33,140,219],"individual":[34],"responses,":[35],"can":[36,58],"provide":[37],"an":[38],"effective":[39,217],"supervision":[40],"signal":[41],"through":[42,101],"relative":[43],"deltas,":[45],"we":[47,75,120],"term":[48],"a":[49,54,86,97,126],"\"weak\"":[50,60,155],"signal.":[51],"This":[52],"motivates":[53],"key":[55],"research":[56],"question:":[57],"multiple":[59,154],"signals":[61,156,168],"be":[62],"constructively":[63],"aggregated":[64],"for":[65,186],"improving":[66],"8B)?":[71],"To":[72,112],"this":[73],"end,":[74],"propose":[76],"Preference":[77],"Delta":[78],"Aggregation":[79],"(PDA),":[80],"first":[82],"framework":[83],"derives":[85],"delta":[88],"each":[90],"pair,":[93],"instantiates":[94],"it":[95],"as":[96,166],"LoRA":[98,110,118],"adapter":[99,132],"learned":[100],"optimization,":[103],"and":[104,147,181,189,197,207],"aggregates":[105],"resulting":[107],"deltas":[108],"via":[109],"merging.":[111],"further":[113,164],"mitigate":[114],"directional":[115],"interference":[116],"during":[117],"merging,":[119],"introduce":[121],"Geometric":[122],"Alignment":[123],"Merging":[124],"(GAM),":[125],"geometry-aware":[127],"merging":[128],"method":[129],"aligns":[131],"subspaces":[133],"before":[134],"aggregation,":[135],"enabling":[136],"more":[137],"robust":[138],"composition":[139,218],"diverse":[141],"deltas.":[142,226],"Evaluations":[143],"on":[144,184],"knowledge":[145,187],"reasoning":[146,188],"agentic":[148,190],"search":[149],"benchmarks":[150],"show":[151],"aggregating":[153],"pushes":[157],"performance":[158],"beyond":[159],"any":[160],"single":[161],"signal,":[162],"with":[163,173],"gains":[165,214],"additional":[167],"are":[169],"incorporated.":[170],"Correspondingly,":[171],"PDA":[172],"GAM":[174],"improves":[175],"by":[179,205],"6.8":[180],"7.3":[182],"points":[183],"average":[185],"search,":[191],"respectively.":[192],"It":[193],"outperforms":[194],"all":[195],"single-delta":[196,203],"multi-delta":[198],"baselines,":[199],"exceeding":[200],"best":[202],"baseline":[204],"2.1":[206],"4.3":[208],"points.":[209],"Further":[210],"analysis":[211],"attributes":[212],"these":[213],"to":[215],"complementary":[220],"capabilities":[221],"encoded":[222],"across":[223],"distinct":[224]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-03T00:00:00"}