{"id":"https://openalex.org/W4409348012","doi":"https://doi.org/10.1609/aaai.v39i22.34558","title":"Preference-Oriented Supervised Fine-Tuning: Favoring Target Model over Aligned Large Language Models","display_name":"Preference-Oriented Supervised Fine-Tuning: Favoring Target Model over Aligned Large Language Models","publication_year":2025,"publication_date":"2025-04-11","ids":{"openalex":"https://openalex.org/W4409348012","doi":"https://doi.org/10.1609/aaai.v39i22.34558"},"language":"en","primary_location":{"id":"doi:10.1609/aaai.v39i22.34558","doi":"https://doi.org/10.1609/aaai.v39i22.34558","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i22.34558","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34558/36713","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"volume":null,"type":"conference-paper","indexed_in":["arxiv","crossref","datacite"],"open_access":{"is_oa":true,"oa_status":"diamond","oa_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34558/36713","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5081094059","display_name":"Yuchen Fan","orcid":"https://orcid.org/0000-0002-2838-7708","observed_orcids":["https://orcid.org/0000-0002-2838-7708"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuchen Fan","raw_affiliation_strings":["Zuoyebang Education Technology (Beijing) Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zuoyebang Education Technology (Beijing) Co., Ltd","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5010955583","display_name":"Yuzhong Hong","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yuzhong Hong","raw_affiliation_strings":["Zuoyebang Education Technology (Beijing) Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zuoyebang Education Technology (Beijing) Co., Ltd","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100411694","display_name":"Qiushi Wang","orcid":"https://orcid.org/0000-0002-1395-5952","observed_orcids":["https://orcid.org/0000-0002-1395-5952"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Qiushi Wang","raw_affiliation_strings":["Zuoyebang Education Technology (Beijing) Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zuoyebang Education Technology (Beijing) Co., Ltd","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5013785156","display_name":"Junwei Bao","orcid":"https://orcid.org/0000-0002-5549-5130","observed_orcids":["https://orcid.org/0000-0002-5549-5130"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Junwei Bao","raw_affiliation_strings":["Zuoyebang Education Technology (Beijing) Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zuoyebang Education Technology (Beijing) Co., Ltd","institution_ids":[]}]},{"author_position":"middle","author":{"id":null,"display_name":"Hongfei Jiang","orcid":null,"observed_orcids":[]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hongfei Jiang","raw_affiliation_strings":["Zuoyebang Education Technology (Beijing) Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zuoyebang Education Technology (Beijing) Co., Ltd","institution_ids":[]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5101423079","display_name":"Yang Song","orcid":"https://orcid.org/0000-0002-7699-5855","observed_orcids":["https://orcid.org/0000-0002-7699-5855"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Yang Song","raw_affiliation_strings":["Zuoyebang Education Technology (Beijing) Co., Ltd"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Zuoyebang Education Technology (Beijing) Co., Ltd","institution_ids":[]}]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":3.7363,"has_fulltext":true,"cited_by_count":4,"citation_normalized_percentile":{"value":0.92865276,"is_in_top_1_percent":false,"is_in_top_10_percent":true},"cited_by_percentile_year":{"min":97,"max":98},"biblio":{"volume":"39","issue":"22","first_page":"23859","last_page":"23867"},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6000999808311462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.6000999808311462,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.16369999945163727,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.14069999754428864,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/large-language-model-alignment","display_name":"large language model alignment","score":0.8309999704360962},{"id":"https://openalex.org/keywords/instruction-following","display_name":"instruction following","score":0.6179999709129333},{"id":"https://openalex.org/keywords/data-filtering","display_name":"data filtering","score":0.6119999885559082},{"id":"https://openalex.org/keywords/preference-optimization","display_name":"preference optimization","score":0.8610000014305115},{"id":"https://openalex.org/keywords/dpo","display_name":"DPO","score":0.8669999837875366}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6972393989562988},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5166640281677246},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.48740485310554504},{"id":"https://openalex.org/C157524613","wikidata":"https://www.wikidata.org/wiki/Q2828883","display_name":"Fine-tuning","level":2,"score":0.4408700466156006},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42506444454193115},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.3490431010723114},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.19796451926231384},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.192192941904068},{"id":"https://openalex.org/C121332964","wikidata":"https://www.wikidata.org/wiki/Q413","display_name":"Physics","level":0,"score":0.15102717280387878},{"id":"https://openalex.org/C109214941","wikidata":"https://www.wikidata.org/wiki/Q18334","display_name":"Particle physics","level":1,"score":0.07201242446899414}],"mesh":[],"locations_count":4,"locations":[{"id":"doi:10.1609/aaai.v39i22.34558","doi":"https://doi.org/10.1609/aaai.v39i22.34558","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i22.34558","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34558/36713","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},{"id":"pmh:oai:arXiv.org:2412.12865","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2412.12865","pdf_url":"https://arxiv.org/pdf/2412.12865","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"pmh:oai:ojs.aaai.org:article/34558","doi":null,"is_oa":false,"landing_page_url":"https://ojs.aaai.org/index.php/AAAI/article/view/34558","pdf_url":null,"source":{"id":"https://openalex.org/S4306400155","display_name":"AAAI Publications (The Association for the Advancement of Artificial Intelligence (AAAI))","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I2801652627","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/I2801652627"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"2159-5399","raw_type":"info:eu-repo/semantics/article"},{"id":"doi:10.48550/arxiv.2412.12865","doi":"https://doi.org/10.48550/arxiv.2412.12865","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2412.12865","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.1609/aaai.v39i22.34558","doi":"https://doi.org/10.1609/aaai.v39i22.34558","is_oa":true,"landing_page_url":"https://doi.org/10.1609/aaai.v39i22.34558","pdf_url":"https://ojs.aaai.org/index.php/AAAI/article/download/34558/36713","source":{"id":"https://openalex.org/S4210191458","display_name":"Proceedings of the AAAI Conference on Artificial Intelligence","issn_l":"2159-5399","issn":["2159-5399","2374-3468"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":["jufo-2","norway-1"],"host_organization":"https://openalex.org/P4310320058","host_organization_name":"Association for the Advancement of Artificial Intelligence","host_organization_lineage":["https://openalex.org/P4310320058"],"host_organization_lineage_names":["Association for the Advancement of Artificial Intelligence"],"type":"conference"},"license":null,"license_id":null,"version":"publishedVersion","is_accepted":true,"is_published":true,"raw_source_name":"Proceedings of the AAAI Conference on Artificial Intelligence","raw_type":"journal-article"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4409348012.pdf","grobid_xml":"https://content.openalex.org/works/W4409348012.grobid-xml"},"referenced_works_count":4,"referenced_works":["https://openalex.org/W2763110165","https://openalex.org/W2913946806","https://openalex.org/W3002104146","https://openalex.org/W4385889714"],"related_works":["https://openalex.org/W4403704388","https://openalex.org/W4412886892","https://openalex.org/W4416035826","https://openalex.org/W4415335523","https://openalex.org/W6979331320","https://openalex.org/W6966625233","https://openalex.org/W6892041672","https://openalex.org/W4404782689","https://openalex.org/W7166879947","https://openalex.org/W4416035061"],"abstract_inverted_index":{"Alignment,":[0],"endowing":[1],"a":[2,33,37,84,100,123],"pre-trained":[3],"Large":[4],"language":[5,29],"model":[6,106,120],"(LLM)":[7],"with":[8,32,76,189],"the":[9,45,58,65,73,77,104,111,118,130,143,147,155,158,161,171],"ability":[10],"to":[11,57,95,121,195],"follow":[12],"instructions,":[13],"is":[14,94],"crucial":[15],"for":[16,64],"its":[17],"real-world":[18],"applications.":[19],"Conventional":[20],"supervised":[21,87],"fine-tuning":[22,88],"(SFT)":[23],"methods":[24,194],"formalize":[25],"it":[26],"as":[27,208],"causal":[28],"modeling":[30],"typically":[31],"cross-entropy":[34],"objective,":[35],"requiring":[36],"large":[38],"amount":[39],"of":[40,47,79,160],"high-quality":[41],"instruction-response":[42],"pairs.":[43],"However,":[44],"quality":[46,78,138],"widely":[48],"used":[49],"SFT":[50,80,97,113,172,191],"datasets":[51,177],"can":[52,186],"not":[53],"be":[54,187],"guaranteed":[55],"due":[56],"high":[59],"cost":[60],"and":[61,67,154,167,178,199],"intensive":[62],"labor":[63],"creation":[66],"maintenance":[68],"in":[69],"practice.":[70],"To":[71],"overcome":[72],"limitations":[74],"associated":[75],"datasets,":[81],"we":[82,182],"introduce":[83],"novel":[85],"preference-oriented":[86],"approach,":[89],"namely":[90],"PoFT.":[91],"The":[92],"intuition":[93],"boost":[96],"by":[98,129,142,202],"imposing":[99],"particular":[101],"preference:":[102],"favoring":[103],"target":[105,119],"over":[107,170],"aligned":[108,131,144],"LLMs":[109],"on":[110,136],"same":[112],"data.":[114],"This":[115],"preference":[116,204],"encourages":[117],"predict":[122],"higher":[124],"likelihood":[125,141],"than":[126],"that":[127,184],"predicted":[128,140],"LLMs,":[132],"incorporating":[133],"assessment":[134],"information":[135],"data":[137,192],"(i.e.,":[139],"LLMs)":[145],"into":[146],"training":[148,176],"process.":[149],"Extensive":[150],"experiments":[151],"are":[152],"conducted,":[153],"results":[156],"validate":[157],"effectiveness":[159],"proposed":[162],"method.":[163],"PoFT":[164,185],"achieves":[165],"stable":[166],"consistent":[168],"improvements":[169],"baselines":[173],"across":[174],"different":[175],"base":[179],"models.":[180],"Moreover,":[181],"prove":[183],"integrated":[188],"existing":[190],"filtering":[193],"achieve":[196],"better":[197],"performance,":[198],"further":[200],"improved":[201],"following":[203],"optimization":[205],"procedures,":[206],"such":[207],"DPO.":[209]},"counts_by_year":[{"year":2026,"cited_by_count":4}],"updated_date":"2026-11-01T11:40:01.658920","created_date":"2025-04-11T00:00:00"}