{"id":"https://openalex.org/W7131681283","doi":"https://doi.org/10.48550/arxiv.2602.21346","title":"Alignment-Weighted DPO: A principled reasoning approach to improve safety alignment","display_name":"Alignment-Weighted DPO: A principled reasoning approach to improve safety alignment","publication_year":2026,"publication_date":"2026-02-24","ids":{"openalex":"https://openalex.org/W7131681283","doi":"https://doi.org/10.48550/arxiv.2602.21346"},"language":null,"primary_location":{"id":"pmh:doi:10.48550/arxiv.2602.21346","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"type":"article","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":null,"any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5126854607","display_name":"Mengxuan Hu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Hu, Mengxuan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120034789","display_name":"Vivek Datla","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Datla, Vivek V.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126947660","display_name":"Anoop Kumar","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Anoop","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5067459053","display_name":"Zihan Guan","orcid":"https://orcid.org/0000-0002-0331-3403"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Guan, Zihan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126937915","display_name":"Sheng Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Sheng","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5066674792","display_name":"Alfy Samuel","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Samuel, Alfy","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5126908474","display_name":"Daben Liu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Daben","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":0.0,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":{"value":0.24102912,"is_in_top_1_percent":false,"is_in_top_10_percent":false},"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.4327000081539154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.4327000081539154,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.16689999401569366,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.13179999589920044,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.7458999752998352},{"id":"https://openalex.org/keywords/construct","display_name":"Construct (python library)","score":0.519599974155426},{"id":"https://openalex.org/keywords/reinforcement-learning","display_name":"Reinforcement learning","score":0.49230000376701355},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.4471000134944916},{"id":"https://openalex.org/keywords/vulnerability","display_name":"Vulnerability (computing)","score":0.40860000252723694},{"id":"https://openalex.org/keywords/deep-learning","display_name":"Deep learning","score":0.30660000443458557}],"concepts":[{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.7458999752998352},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.703000009059906},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.619700014591217},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.5231999754905701},{"id":"https://openalex.org/C2780801425","wikidata":"https://www.wikidata.org/wiki/Q5164392","display_name":"Construct (python library)","level":2,"score":0.519599974155426},{"id":"https://openalex.org/C97541855","wikidata":"https://www.wikidata.org/wiki/Q830687","display_name":"Reinforcement learning","level":2,"score":0.49230000376701355},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.4471000134944916},{"id":"https://openalex.org/C95713431","wikidata":"https://www.wikidata.org/wiki/Q631425","display_name":"Vulnerability (computing)","level":2,"score":0.40860000252723694},{"id":"https://openalex.org/C108583219","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep learning","level":2,"score":0.30660000443458557},{"id":"https://openalex.org/C195344581","wikidata":"https://www.wikidata.org/wiki/Q2555318","display_name":"Automated reasoning","level":2,"score":0.27649998664855957},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.25870001316070557},{"id":"https://openalex.org/C115086926","wikidata":"https://www.wikidata.org/wiki/Q17004651","display_name":"Causal reasoning","level":3,"score":0.25760000944137573}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:doi:10.48550/arxiv.2602.21346","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},{"id":"doi:10.48550/arxiv.2602.21346","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.21346","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:doi:10.48550/arxiv.2602.21346","is_oa":true,"landing_page_url":null,"pdf_url":null,"source":{"id":"https://openalex.org/S4406922384","display_name":"Open MIND","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":null,"host_organization_name":null,"host_organization_lineage":[],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Article"},"sustainable_development_goals":[{"score":0.7415385246276855,"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Recent":[0],"advances":[1],"in":[2,118,129],"alignment":[3,59,83,185],"techniques":[4],"such":[5],"as":[6],"Supervised":[7],"Fine-Tuning":[8],"(SFT),":[9],"Reinforcement":[10],"Learning":[11],"from":[12,57],"Human":[13],"Feedback":[14],"(RLHF),":[15],"and":[16,89,101,153,164,176],"Direct":[17],"Preference":[18],"Optimization":[19],"(DPO)":[20],"have":[21],"improved":[22],"the":[23,138,151],"safety":[24,175],"of":[25,142],"large":[26],"language":[27],"models":[28,112],"(LLMs).":[29],"However,":[30],"these":[31],"LLMs":[32],"remain":[33],"vulnerable":[34],"to":[35,113,150,167],"jailbreak":[36,169],"attacks":[37],"that":[38,53,61,97,180],"disguise":[39],"harmful":[40,67],"intent":[41],"through":[42,84],"indirect":[43],"or":[44],"deceptive":[45],"phrasing.":[46],"Using":[47],"causal":[48],"intervention,":[49],"we":[50,80,132],"empirically":[51],"demonstrate":[52],"this":[54,78,109],"vulnerability":[55],"stems":[56],"shallow":[58],"mechanisms":[60],"lack":[62],"deep":[63],"reasoning,":[64,119],"often":[65],"rejecting":[66],"prompts":[68,103],"without":[69],"truly":[70],"understanding":[71],"why":[72],"they":[73],"are":[74],"harmful.":[75],"To":[76],"mitigate":[77],"vulnerability,":[79],"propose":[81],"enhancing":[82],"reasoning-aware":[85],"post-training.":[86],"We":[87],"construct":[88],"release":[90],"a":[91],"novel":[92],"Chain-of-Thought":[93],"(CoT)":[94],"fine-tuning":[95],"dataset":[96,110],"includes":[98],"both":[99],"utility-oriented":[100],"safety-critical":[102],"with":[104],"step-by-step":[105],"rationales.":[106],"Fine-tuning":[107],"on":[108],"encourages":[111],"produce":[114],"principled":[115],"refusals":[116],"grounded":[117],"outperforming":[120],"standard":[121],"SFT":[122],"baselines.":[123],"Furthermore,":[124],"inspired":[125],"by":[126,145],"failure":[127],"patterns":[128],"CoT":[130],"fine-tuning,":[131],"introduce":[133],"Alignment-Weighted":[134],"DPO,":[135],"which":[136],"targets":[137],"most":[139],"problematic":[140],"parts":[141],"an":[143],"output":[144],"assigning":[146],"different":[147],"preference":[148],"weights":[149],"reasoning":[152],"final-answer":[154],"segments.":[155],"This":[156],"produces":[157],"finer-grained,":[158],"targeted":[159],"updates":[160],"than":[161],"vanilla":[162],"DPO":[163],"improves":[165,184],"robustness":[166,186],"diverse":[168],"strategies.":[170],"Extensive":[171],"experiments":[172],"across":[173],"multiple":[174],"utility":[177],"benchmarks":[178],"show":[179],"our":[181],"method":[182],"consistently":[183],"while":[187],"maintaining":[188],"overall":[189],"model":[190],"utility.":[191]},"counts_by_year":[],"updated_date":"2026-07-15T18:14:33.161393","created_date":"2026-02-27T00:00:00"}