{"id":"https://openalex.org/W7214804461","doi":"https://doi.org/10.48550/arxiv.2609.35044","title":"BA-DPO: Bias-Adjusted Direct Preference Optimization for Language Model Alignment","display_name":"BA-DPO: Bias-Adjusted Direct Preference Optimization for Language Model Alignment","publication_year":2026,"publication_date":"2026-09-28","ids":{"openalex":"https://openalex.org/W7214804461","doi":"https://doi.org/10.48550/arxiv.2609.35044"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2609.35044","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35044","pdf_url":"https://arxiv.org/pdf/2609.35044","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2609.35044","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088946359","display_name":"Antonio Ferrara","orcid":"https://orcid.org/0000-0002-1921-8304","observed_orcids":["https://orcid.org/0000-0002-1921-8304","https://orcid.org/0000-0002-0784-1115"]},"institutions":[{"id":"https://openalex.org/I4210144624","display_name":"Intesa Sanpaolo (Italy)","ror":"https://ror.org/04a2yhg09","country_code":"IT","type":"company","lineage":["https://openalex.org/I4210144624"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Ferrara, Antonio","raw_affiliation_strings":["Intesa Sanpaolo AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intesa Sanpaolo AI Research","institution_ids":["https://openalex.org/I4210144624"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5012673105","display_name":"Alberto Rumi","orcid":"https://orcid.org/0000-0001-7426-3012","observed_orcids":["https://orcid.org/0000-0001-7426-3012"]},"institutions":[{"id":"https://openalex.org/I4210144624","display_name":"Intesa Sanpaolo (Italy)","ror":"https://ror.org/04a2yhg09","country_code":"IT","type":"company","lineage":["https://openalex.org/I4210144624"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Rumi, Alberto","raw_affiliation_strings":["Intesa Sanpaolo AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intesa Sanpaolo AI Research","institution_ids":["https://openalex.org/I4210144624"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5022272025","display_name":"Francesco Bonchi","orcid":"https://orcid.org/0000-0001-9464-8315","observed_orcids":["https://orcid.org/0000-0001-9464-8315"]},"institutions":[{"id":"https://openalex.org/I4210144624","display_name":"Intesa Sanpaolo (Italy)","ror":"https://ror.org/04a2yhg09","country_code":"IT","type":"company","lineage":["https://openalex.org/I4210144624"]}],"countries":["IT"],"is_corresponding":false,"raw_author_name":"Bonchi, Francesco","raw_affiliation_strings":["Intesa Sanpaolo AI Research"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Intesa Sanpaolo AI Research","institution_ids":["https://openalex.org/I4210144624"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I4210144624"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.8353999853134155,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.8353999853134155,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.03739999979734421,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.03350000083446503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/preference-based-alignment","display_name":"preference-based alignment","score":0.5899999737739563},{"id":"https://openalex.org/keywords/language-model-alignment","display_name":"language model alignment","score":0.8050000071525574},{"id":"https://openalex.org/keywords/annotator-bias","display_name":"annotator bias","score":0.8450000286102295},{"id":"https://openalex.org/keywords/statistical-parity","display_name":"statistical parity","score":0.8479999899864197},{"id":"https://openalex.org/keywords/length-bias","display_name":"length bias","score":0.6269999742507935},{"id":"https://openalex.org/keywords/low-rank-adaptation","display_name":"low-rank adaptation","score":0.8190000057220459}],"concepts":[{"id":"https://openalex.org/C184898388","wikidata":"https://www.wikidata.org/wiki/Q1435712","display_name":"Pairwise comparison","level":2,"score":0.7364000082015991},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7128999829292297},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.617900013923645},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.567300021648407},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.553600013256073},{"id":"https://openalex.org/C204321447","wikidata":"https://www.wikidata.org/wiki/Q30642","display_name":"Natural language processing","level":1,"score":0.5457000136375427},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.5295000076293945},{"id":"https://openalex.org/C2777027219","wikidata":"https://www.wikidata.org/wiki/Q1284190","display_name":"Constant (computer programming)","level":2,"score":0.4368000030517578},{"id":"https://openalex.org/C81917197","wikidata":"https://www.wikidata.org/wiki/Q628760","display_name":"Selection (genetic algorithm)","level":2,"score":0.4016999900341034},{"id":"https://openalex.org/C114289077","wikidata":"https://www.wikidata.org/wiki/Q3284399","display_name":"Statistical model","level":2,"score":0.3919999897480011},{"id":"https://openalex.org/C157972887","wikidata":"https://www.wikidata.org/wiki/Q463359","display_name":"Convex optimization","level":3,"score":0.36070001125335693},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3328999876976013},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3328999876976013},{"id":"https://openalex.org/C2776299755","wikidata":"https://www.wikidata.org/wiki/Q432449","display_name":"Carry (investment)","level":2,"score":0.3248000144958496},{"id":"https://openalex.org/C40969351","wikidata":"https://www.wikidata.org/wiki/Q3516228","display_name":"Word error rate","level":2,"score":0.29580000042915344},{"id":"https://openalex.org/C2776818064","wikidata":"https://www.wikidata.org/wiki/Q829903","display_name":"Agreement","level":2,"score":0.2858999967575073},{"id":"https://openalex.org/C195324797","wikidata":"https://www.wikidata.org/wiki/Q33742","display_name":"Natural language","level":2,"score":0.2833999991416931},{"id":"https://openalex.org/C2776359362","wikidata":"https://www.wikidata.org/wiki/Q2145286","display_name":"Representation (politics)","level":3,"score":0.2685999870300293}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2609.35044","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35044","pdf_url":"https://arxiv.org/pdf/2609.35044","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2609.35044","doi":"https://doi.org/10.48550/arxiv.2609.35044","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2609.35044","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2609.35044","doi":null,"is_oa":true,"landing_page_url":"https://arxiv.org/abs/2609.35044","pdf_url":"https://arxiv.org/pdf/2609.35044","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7214804461.pdf","grobid_xml":"https://content.openalex.org/works/W7214804461.grobid-xml"},"referenced_works_count":17,"referenced_works":["https://openalex.org/W2970019270","https://openalex.org/W2984256198","https://openalex.org/W3035139434","https://openalex.org/W4223908421","https://openalex.org/W4387561528","https://openalex.org/W4389519493","https://openalex.org/W4391157361","https://openalex.org/W4401306886","https://openalex.org/W4403623502","https://openalex.org/W4405655184","https://openalex.org/W4406549323","https://openalex.org/W4412944612","https://openalex.org/W4414274825","https://openalex.org/W4416338909","https://openalex.org/W6891904700","https://openalex.org/W7102392001","https://openalex.org/W7162638998"],"related_works":["https://openalex.org/W7140127495","https://openalex.org/W4402670121","https://openalex.org/W4405173242","https://openalex.org/W7162662891","https://openalex.org/W4412889417","https://openalex.org/W7128515446","https://openalex.org/W4415109957","https://openalex.org/W7161160542","https://openalex.org/W4417114736","https://openalex.org/W7123946233"],"abstract_inverted_index":{"Preference-based":[0],"alignment":[1],"methods":[2,62],"such":[3],"as":[4],"Direct":[5],"Preference":[6],"Optimization":[7],"(DPO)":[8],"use":[9,152],"pairwise":[10],"preferences":[11],"labeled":[12],"by":[13,138],"human":[14],"annotators":[15,21,191],"to":[16,71,123,153,158,175,182],"fine-tune":[17],"language":[18,40],"models.":[19],"However,":[20],"carry":[22],"systematic":[23,52],"biases":[24,53,73],"toward":[25,74,97],"some":[26],"attributes:":[27],"a":[28,32,37,39,42,86,100,124,147,155,162,172],"name":[29],"that":[30,90,105,115,185],"signals":[31],"gender":[33],"or":[34,45,66,146],"an":[35],"ethnicity,":[36],"persona,":[38],"variety,":[41],"formatting":[43],"convention,":[44],"length.":[46],"If":[47],"not":[48],"properly":[49],"addressed,":[50],"these":[51],"can":[54],"be":[55],"absorbed":[56],"and":[57,114,178,206,217],"amplified":[58],"during":[59],"alignment.":[60],"Existing":[61],"address":[63,78],"length":[64],"bias":[65,93,112,121],"annotator":[67,96],"disagreement,":[68],"but":[69],"fail":[70],"eliminate":[72],"arbitrary":[75],"attributes.":[76],"To":[77],"this":[79],"limitation,":[80],"we":[81,151],"propose":[82],"Bias-Adjusted":[83],"DPO":[84,89,167,216],"(BA-DPO),":[85],"generalization":[87],"of":[88,142,184,196],"adds":[91],"one":[92],"parameter":[94],"per":[95],"responses":[98],"carrying":[99],"declared":[101],"attribute.":[102],"We":[103],"prove":[104],"the":[106,111,116,133,140,143,169],"objective":[107],"is":[108,130],"convex":[109],"in":[110,220],"parameters":[113],"votes":[117],"identify":[118],"each":[119],"annotator's":[120],"up":[122],"shared":[125],"constant.":[126],"The":[127],"remaining":[128],"constant":[129],"what":[131],"fixes":[132],"aligned":[134],"model's":[135],"attribute":[136,170],"rate:":[137],"default":[139],"rate":[141],"reference":[144],"model,":[145],"target":[148],"rate,":[149],"which":[150],"bring":[154],"biased":[156],"policy":[157],"statistical":[159],"parity.":[160],"On":[161],"corpus":[163],"with":[164,189,203,209],"planted":[165],"biases,":[166],"drives":[168],"from":[171],"balanced":[173],"start":[174],"probability":[176],"0.96":[177],"BA-DPO":[179],"removes":[180,193],"81":[181],"95\\%":[183],"shift;":[186],"on":[187],"MultiPref":[188],"real":[190],"it":[192],"about":[194],"half":[195],"DPO's":[197],"lengthening.":[198],"Both":[199],"hold":[200],"at":[201,207,211],"0.5B":[202],"full":[204],"fine-tuning":[205],"8B":[208],"LoRA,":[210],"no":[212,218],"higher":[213],"KL":[214],"than":[215],"loss":[219],"judged":[221],"quality.":[222]},"counts_by_year":[],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2026-09-30T00:00:00"}
