{"id":"https://openalex.org/W4403564074","doi":"https://doi.org/10.48550/arxiv.2410.08847","title":"Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization","display_name":"Unintentional Unalignment: Likelihood Displacement in Direct Preference Optimization","publication_year":2024,"publication_date":"2024-10-11","ids":{"openalex":"https://openalex.org/W4403564074","doi":"https://doi.org/10.48550/arxiv.2410.08847"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2410.08847","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.08847","pdf_url":"https://arxiv.org/pdf/2410.08847","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2410.08847","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5030945372","display_name":"Noam Razin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Razin, Noam","raw_affiliation_strings":["Princeton ORFE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton ORFE","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5077971483","display_name":"Sadhika Malladi","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Malladi, Sadhika","raw_affiliation_strings":["Princeton ORFE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton ORFE","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Bhaskar, Adithya","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Bhaskar, Adithya","raw_affiliation_strings":["Princeton ORFE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton ORFE","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5051064208","display_name":"Danqi Chen","orcid":"https://orcid.org/0000-0002-6226-6838","observed_orcids":["https://orcid.org/0000-0002-6226-6838","https://orcid.org/0000-0001-5308-2634"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Chen, Danqi","raw_affiliation_strings":["Princeton ORFE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton ORFE","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5103209777","display_name":"Sanjeev Arora","orcid":"https://orcid.org/0000-0002-8636-9970","observed_orcids":["https://orcid.org/0000-0002-8636-9970"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Arora, Sanjeev","raw_affiliation_strings":["Princeton ORFE"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton ORFE","institution_ids":["https://openalex.org/I20089843"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5103247445","display_name":"Boris Hanin","orcid":"https://orcid.org/0000-0003-1614-0131","observed_orcids":["https://orcid.org/0000-0003-1614-0131"]},"institutions":[{"id":"https://openalex.org/I20089843","display_name":"Princeton University","ror":"https://ror.org/00hx57361","country_code":"US","type":"education","lineage":["https://openalex.org/I20089843"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Hanin, Boris","raw_affiliation_strings":["Princeton Language and Intelligence , Princeton University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Princeton Language and Intelligence , Princeton University","institution_ids":["https://openalex.org/I20089843"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":1,"corresponding_author_ids":[],"corresponding_institution_ids":["https://openalex.org/I20089843"],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":53,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.29420000314712524,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.29420000314712524,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.2003999948501587,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.10750000178813934,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/direct-preference-optimization","display_name":"direct preference optimization","score":0.7990000247955322},{"id":"https://openalex.org/keywords/preference-optimization","display_name":"preference optimization","score":0.5519999861717224},{"id":"https://openalex.org/keywords/language-model-alignment","display_name":"language model alignment","score":0.8709999918937683},{"id":"https://openalex.org/keywords/refusal-behavior","display_name":"refusal behavior","score":0.41100001335144043},{"id":"https://openalex.org/keywords/training-data-filtering","display_name":"training data filtering","score":0.5950000286102295}],"concepts":[{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.6579390168190002},{"id":"https://openalex.org/C107551265","wikidata":"https://www.wikidata.org/wiki/Q1458245","display_name":"Displacement (psychology)","level":2,"score":0.5445073843002319},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3726017475128174},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.2536389231681824},{"id":"https://openalex.org/C15744967","wikidata":"https://www.wikidata.org/wiki/Q9418","display_name":"Psychology","level":0,"score":0.24252206087112427},{"id":"https://openalex.org/C105795698","wikidata":"https://www.wikidata.org/wiki/Q12483","display_name":"Statistics","level":1,"score":0.22460639476776123},{"id":"https://openalex.org/C542102704","wikidata":"https://www.wikidata.org/wiki/Q183257","display_name":"Psychotherapist","level":1,"score":0.0}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2410.08847","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.08847","pdf_url":"https://arxiv.org/pdf/2410.08847","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2410.08847","doi":"https://doi.org/10.48550/arxiv.2410.08847","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2410.08847","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2410.08847","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2410.08847","pdf_url":"https://arxiv.org/pdf/2410.08847","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G2039971938","display_name":"Collaborative Research: RI:Medium:MoDL:Mathematical and Conceptual Understanding of Large Language Models","funder_award_id":"2211779","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G215710341","display_name":"Collaborative Research: Probabilistic, Geometric, and Topological Analysis of Neural Networks, From Theory to Applications","funder_award_id":"2133806","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G4247196128","display_name":"Random Neural Networks","funder_award_id":"1855684","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G6671297155","display_name":null,"funder_award_id":"CAREER","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"},{"id":"https://openalex.org/G8207267925","display_name":"CAREER: Random Neural Nets and Random Matrix Products","funder_award_id":"2143754","funder_id":"https://openalex.org/F4320306076","funder_display_name":"National Science Foundation"}],"funders":[{"id":"https://openalex.org/F4320306076","display_name":"National Science Foundation","ror":"https://ror.org/021nxhr62"},{"id":"https://openalex.org/F4320332180","display_name":"Defense Advanced Research Projects Agency","ror":"https://ror.org/02caytj08"},{"id":"https://openalex.org/F4320337345","display_name":"Office of Naval Research","ror":"https://ror.org/00rk2pe57"}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4403564074.pdf","grobid_xml":"https://content.openalex.org/works/W4403564074.grobid-xml"},"referenced_works_count":44,"referenced_works":["https://openalex.org/W2973379954","https://openalex.org/W2980282514","https://openalex.org/W3086677109","https://openalex.org/W4223908421","https://openalex.org/W4311991106","https://openalex.org/W4312050653","https://openalex.org/W4327810158","https://openalex.org/W4384111857","https://openalex.org/W4385571189","https://openalex.org/W4387994989","https://openalex.org/W4388890574","https://openalex.org/W4390214435","https://openalex.org/W4390723197","https://openalex.org/W4390963024","https://openalex.org/W4392019816","https://openalex.org/W4392822465","https://openalex.org/W4393740941","https://openalex.org/W4393967327","https://openalex.org/W4394654134","https://openalex.org/W4394906585","https://openalex.org/W4394972204","https://openalex.org/W4395444023","https://openalex.org/W4396650433","https://openalex.org/W4399198758","https://openalex.org/W4399198767","https://openalex.org/W4399317978","https://openalex.org/W4399991953","https://openalex.org/W4400714246","https://openalex.org/W4401042550","https://openalex.org/W4401306886","https://openalex.org/W4402671236","https://openalex.org/W4402683786","https://openalex.org/W4404782689","https://openalex.org/W4406071657","https://openalex.org/W4415795301","https://openalex.org/W4415795446","https://openalex.org/W4415795501","https://openalex.org/W4415795685","https://openalex.org/W4415796092","https://openalex.org/W4415797532","https://openalex.org/W4415798774","https://openalex.org/W4415799047","https://openalex.org/W4415800425","https://openalex.org/W7196966086"],"related_works":["https://openalex.org/W4407425735","https://openalex.org/W4402670121","https://openalex.org/W7140127495","https://openalex.org/W4416610404","https://openalex.org/W4415795724","https://openalex.org/W4416285728","https://openalex.org/W4415109957","https://openalex.org/W7197026380","https://openalex.org/W4404400430","https://openalex.org/W6891904700"],"abstract_inverted_index":{"Direct":[0],"Preference":[1],"Optimization":[2],"(DPO)":[3],"and":[4,58],"its":[5],"variants":[6],"are":[7,20],"increasingly":[8],"used":[9],"for":[10,216],"aligning":[11,110],"language":[12],"models":[13],"with":[14,85,212],"human":[15],"preferences.":[16],"Although":[17],"these":[18,193],"methods":[19],"designed":[21],"to":[22,26,33,83,96,113,125,135,147,184],"teach":[23],"a":[24,90,94,166,188],"model":[25,95,112],"generate":[27],"preferred":[28,44,81,132],"responses":[29,45,82,84,134,137],"more":[30],"frequently":[31],"relative":[32],"dispreferred":[34],"responses,":[35],"prior":[36],"work":[37,52],"has":[38],"observed":[39],"that":[40,71,119,152,159],"the":[41,56,104,111,140,174,207,220],"likelihood":[42,67,72,153,185],"of":[43,60,106,143,209],"often":[46],"decreases":[47],"during":[48],"training.":[49],"The":[50],"current":[51],"sheds":[53],"light":[54],"on":[55],"causes":[57],"implications":[59],"this":[61],"counter-intuitive":[62],"phenomenon,":[63],"which":[64,179,217],"we":[65,117,218],"term":[66],"displacement.":[68],"We":[69,149],"demonstrate":[70],"displacement":[73,121,154,186],"can":[74,101,122],"be":[75],"catastrophic,":[76],"shifting":[77,128],"probability":[78,105,129],"mass":[79,130],"from":[80,131,145],"an":[86],"opposite":[87],"meaning.":[88],"As":[89],"simple":[91],"example,":[92],"training":[93,180],"prefer":[97],"$\\texttt{No}$":[98],"over":[99],"$\\texttt{Never}$":[100],"sharply":[102],"increase":[103],"$\\texttt{Yes}$.":[107],"Moreover,":[108],"when":[109],"refuse":[114],"unsafe":[115],"prompts,":[116],"show":[118],"such":[120],"unintentionally":[123],"lead":[124],"unalignment,":[126],"by":[127,157,165],"refusal":[133,141],"harmful":[136],"(e.g.,":[138],"reducing":[139],"rate":[142],"Llama-3-8B-Instruct":[144],"74.4%":[146],"33.4%).":[148],"theoretically":[150],"characterize":[151],"is":[155],"driven":[156],"preferences":[158],"induce":[160],"similar":[161],"embeddings,":[162],"as":[163],"measured":[164],"centered":[167],"hidden":[168],"embedding":[169],"similarity":[170],"(CHES)":[171],"score.":[172],"Empirically,":[173],"CHES":[175,221],"score":[176,222],"enables":[177],"identifying":[178],"samples":[181,194],"contribute":[182],"most":[183],"in":[187,199],"given":[189],"dataset.":[190],"Filtering":[191],"out":[192],"effectively":[195],"mitigated":[196],"unintentional":[197],"unalignment":[198],"our":[200,204],"experiments.":[201],"More":[202],"broadly,":[203],"results":[205],"highlight":[206],"importance":[208],"curating":[210],"data":[211],"sufficiently":[213],"distinct":[214],"preferences,":[215],"believe":[219],"may":[223],"prove":[224],"valuable.":[225]},"counts_by_year":[{"year":2026,"cited_by_count":21},{"year":2025,"cited_by_count":29},{"year":2024,"cited_by_count":3}],"updated_date":"2026-11-07T11:25:07.637180","created_date":"2024-10-20T00:00:00"}