{"id":"https://openalex.org/W7153263265","doi":"https://doi.org/10.48550/arxiv.2604.07754","title":"The Art of (Mis)alignment: How Fine-Tuning Methods Effectively Misalign and Realign LLMs in Post-Training","display_name":"The Art of (Mis)alignment: How Fine-Tuning Methods Effectively Misalign and Realign LLMs in Post-Training","publication_year":2026,"publication_date":"2026-04-09","ids":{"openalex":"https://openalex.org/W7153263265","doi":"https://doi.org/10.48550/arxiv.2604.07754"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2604.07754","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07754","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2604.07754","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5100422174","display_name":"Rui Zhang","orcid":"https://orcid.org/0000-0003-3404-6800"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Rui","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133394869","display_name":"Hongwei Li","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Li, Hongwei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133355101","display_name":"Yun Shen","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Yun","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102726629","display_name":"Xinyue Shen","orcid":"https://orcid.org/0009-0006-9954-587X"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Shen, Xinyue","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133383949","display_name":"Wenbo Jiang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Jiang, Wenbo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133327130","display_name":"Guowen Xu","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Xu, Guowen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5100638749","display_name":"Yang Liu","orcid":"https://orcid.org/0000-0002-4795-9169"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Liu, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5133327407","display_name":"Michael Backes","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Backes, Michael","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5100354708","display_name":"Yang Zhang","orcid":"https://orcid.org/0000-0002-6804-1840"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yang","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11689","display_name":"Adversarial Robustness in Machine Learning","score":0.9847999811172485,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10883","display_name":"Ethics and Social Impacts of AI","score":0.0026000000070780516,"subfield":{"id":"https://openalex.org/subfields/3311","display_name":"Safety Research"},"field":{"id":"https://openalex.org/fields/33","display_name":"Social Sciences"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}},{"id":"https://openalex.org/T12026","display_name":"Explainable Artificial Intelligence (XAI)","score":0.002099999925121665,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/exploit","display_name":"Exploit","score":0.7113000154495239},{"id":"https://openalex.org/keywords/software-deployment","display_name":"Software deployment","score":0.659500002861023},{"id":"https://openalex.org/keywords/preference","display_name":"Preference","score":0.621399998664856},{"id":"https://openalex.org/keywords/adversarial-system","display_name":"Adversarial system","score":0.546500027179718},{"id":"https://openalex.org/keywords/odds","display_name":"Odds","score":0.5105000138282776},{"id":"https://openalex.org/keywords/limiting","display_name":"Limiting","score":0.4162999987602234},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3702000081539154},{"id":"https://openalex.org/keywords/work","display_name":"Work (physics)","score":0.34630000591278076}],"concepts":[{"id":"https://openalex.org/C165696696","wikidata":"https://www.wikidata.org/wiki/Q11287","display_name":"Exploit","level":2,"score":0.7113000154495239},{"id":"https://openalex.org/C105339364","wikidata":"https://www.wikidata.org/wiki/Q2297740","display_name":"Software deployment","level":2,"score":0.659500002861023},{"id":"https://openalex.org/C2781249084","wikidata":"https://www.wikidata.org/wiki/Q908656","display_name":"Preference","level":2,"score":0.621399998664856},{"id":"https://openalex.org/C112930515","wikidata":"https://www.wikidata.org/wiki/Q4389547","display_name":"Risk analysis (engineering)","level":1,"score":0.6208000183105469},{"id":"https://openalex.org/C38652104","wikidata":"https://www.wikidata.org/wiki/Q3510521","display_name":"Computer security","level":1,"score":0.6010000109672546},{"id":"https://openalex.org/C37736160","wikidata":"https://www.wikidata.org/wiki/Q1801315","display_name":"Adversarial system","level":2,"score":0.546500027179718},{"id":"https://openalex.org/C143095724","wikidata":"https://www.wikidata.org/wiki/Q515895","display_name":"Odds","level":3,"score":0.5105000138282776},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.42260000109672546},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.4162999987602234},{"id":"https://openalex.org/C144133560","wikidata":"https://www.wikidata.org/wiki/Q4830453","display_name":"Business","level":0,"score":0.38580000400543213},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3702000081539154},{"id":"https://openalex.org/C108827166","wikidata":"https://www.wikidata.org/wiki/Q175975","display_name":"Internet privacy","level":1,"score":0.36410000920295715},{"id":"https://openalex.org/C18762648","wikidata":"https://www.wikidata.org/wiki/Q42213","display_name":"Work (physics)","level":2,"score":0.34630000591278076},{"id":"https://openalex.org/C152163801","wikidata":"https://www.wikidata.org/wiki/Q2146057","display_name":"Residual risk","level":2,"score":0.34470000863075256},{"id":"https://openalex.org/C127413603","wikidata":"https://www.wikidata.org/wiki/Q11023","display_name":"Engineering","level":0,"score":0.3370000123977661},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.31610000133514404},{"id":"https://openalex.org/C49937458","wikidata":"https://www.wikidata.org/wiki/Q2599292","display_name":"Probabilistic logic","level":2,"score":0.30309998989105225},{"id":"https://openalex.org/C12174686","wikidata":"https://www.wikidata.org/wiki/Q1058438","display_name":"Risk assessment","level":2,"score":0.29899999499320984},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.29179999232292175},{"id":"https://openalex.org/C140547941","wikidata":"https://www.wikidata.org/wiki/Q7797194","display_name":"Threat model","level":2,"score":0.2906999886035919},{"id":"https://openalex.org/C153517567","wikidata":"https://www.wikidata.org/wiki/Q26090","display_name":"Mechanism design","level":2,"score":0.27799999713897705},{"id":"https://openalex.org/C3017944768","wikidata":"https://www.wikidata.org/wiki/Q1450463","display_name":"Poison control","level":2,"score":0.26089999079704285},{"id":"https://openalex.org/C39549134","wikidata":"https://www.wikidata.org/wiki/Q133080","display_name":"Public relations","level":1,"score":0.25780001282691956},{"id":"https://openalex.org/C32896092","wikidata":"https://www.wikidata.org/wiki/Q189447","display_name":"Risk management","level":2,"score":0.251800000667572},{"id":"https://openalex.org/C155512373","wikidata":"https://www.wikidata.org/wiki/Q287450","display_name":"Residual","level":2,"score":0.25040000677108765}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2604.07754","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07754","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2604.07754","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2604.07754","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Peace, Justice and strong institutions","id":"https://metadata.un.org/sdg/16","score":0.6007519960403442}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"The":[0],"deployment":[1,171],"of":[2,72,77,83,134,144,172],"large":[3],"language":[4],"models":[5],"(LLMs)":[6],"raises":[7],"significant":[8],"ethical":[9],"and":[10,23,80,92,110,148,160],"safety":[11,22,32,54,162],"concerns.":[12],"While":[13,112],"LLM":[14],"alignment":[15,163],"techniques":[16,29],"are":[17],"adopted":[18],"to":[19,30,47,57,165],"improve":[20],"model":[21,135],"trustworthiness,":[24],"adversaries":[25],"can":[26],"exploit":[27],"these":[28],"undermine":[31],"for":[33,121,157],"malicious":[34],"purposes,":[35],"resulting":[36],"in":[37,75,128,169],"\\emph{misalignment}.":[38],"Misaligned":[39],"LLMs":[40],"may":[41],"be":[42],"published":[43],"on":[44],"open":[45],"platforms":[46],"magnify":[48],"harm.":[49],"To":[50],"address":[51],"this,":[52],"additional":[53],"alignment,":[55],"referred":[56],"as":[58],"\\emph{realignment},":[59],"is":[60,118,176],"necessary":[61],"before":[62],"deploying":[63],"untrusted":[64],"third-party":[65],"LLMs.":[66,173],"This":[67],"study":[68],"explores":[69],"the":[70,81,132,155,170],"efficacy":[71],"fine-tuning":[73],"methods":[74,97],"terms":[76],"misalignment,":[78,122],"realignment,":[79,129],"effects":[82,143],"their":[84],"interplay.":[85],"By":[86],"evaluating":[87],"four":[88,99],"Supervised":[89],"Fine-Tuning":[90,95],"(SFT)":[91],"two":[93],"Preference":[94,115,124],"(PFT)":[96],"across":[98],"popular":[100],"safety-aligned":[101],"LLMs,":[102],"we":[103,138],"reveal":[104],"a":[105],"mechanism":[106],"asymmetry":[107],"between":[108],"attack":[109],"defense.":[111],"Odds":[113],"Ratio":[114],"Optimization":[116,125],"(ORPO)":[117],"most":[119],"effective":[120],"Direct":[123],"(DPO)":[126],"excels":[127],"albeit":[130],"at":[131,178],"expense":[133],"utility.":[136],"Additionally,":[137],"identify":[139],"model-specific":[140],"resistance,":[141],"residual":[142],"multi-round":[145],"adversarial":[146],"dynamics,":[147],"other":[149],"noteworthy":[150],"findings.":[151],"These":[152],"findings":[153],"highlight":[154],"need":[156],"robust":[158],"safeguards":[159],"customized":[161],"strategies":[164],"mitigate":[166],"potential":[167],"risks":[168],"Our":[174],"code":[175],"available":[177],"https://github.com/zhangrui4041/The-Art-of-Mis-alignment.":[179]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-04-11T00:00:00"}