{"id":"https://openalex.org/W7165630237","doi":"https://doi.org/10.48550/arxiv.2606.20814","title":"What Shapes Emergent Misalignment? Insights from Training Dynamics, Model Priors, and Data","display_name":"What Shapes Emergent Misalignment? Insights from Training Dynamics, Model Priors, and Data","publication_year":2026,"publication_date":"2026-06-18","ids":{"openalex":"https://openalex.org/W7165630237","doi":"https://doi.org/10.48550/arxiv.2606.20814"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.20814","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.20814","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.20814","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5139165717","display_name":"Yuchen Zhang","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zhang, Yuchen","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5126187517","display_name":"Anietta Weckauff","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Weckauff, Anietta","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5061696723","display_name":"Diego Garc\u00eda-Olano","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Garcia-Olano, Diego","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5139196629","display_name":"Maksym Andriushchenko","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Andriushchenko, Maksym","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5766000151634216,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.5766000151634216,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.1111999973654747,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10656","display_name":"Child and Animal Learning Development","score":0.03880000114440918,"subfield":{"id":"https://openalex.org/subfields/3204","display_name":"Developmental and Educational Psychology"},"field":{"id":"https://openalex.org/fields/32","display_name":"Psychology"},"domain":{"id":"https://openalex.org/domains/2","display_name":"Social Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/maxima-and-minima","display_name":"Maxima and minima","score":0.761900007724762},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.7311999797821045},{"id":"https://openalex.org/keywords/subspace-topology","display_name":"Subspace topology","score":0.6769000291824341},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.6164000034332275},{"id":"https://openalex.org/keywords/similarity","display_name":"Similarity (geometry)","score":0.5788999795913696},{"id":"https://openalex.org/keywords/standard-deviation","display_name":"Standard deviation","score":0.4196999967098236},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.38999998569488525}],"concepts":[{"id":"https://openalex.org/C186633575","wikidata":"https://www.wikidata.org/wiki/Q845060","display_name":"Maxima and minima","level":2,"score":0.761900007724762},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.7311999797821045},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.6769000291824341},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.6164000034332275},{"id":"https://openalex.org/C103278499","wikidata":"https://www.wikidata.org/wiki/Q254465","display_name":"Similarity (geometry)","level":3,"score":0.5788999795913696},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5782999992370605},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5630000233650208},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.492000013589859},{"id":"https://openalex.org/C22679943","wikidata":"https://www.wikidata.org/wiki/Q159375","display_name":"Standard deviation","level":2,"score":0.4196999967098236},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.38999998569488525},{"id":"https://openalex.org/C33676613","wikidata":"https://www.wikidata.org/wiki/Q13415176","display_name":"Dimension (graph theory)","level":2,"score":0.3061999976634979},{"id":"https://openalex.org/C50335755","wikidata":"https://www.wikidata.org/wiki/Q483247","display_name":"Phenomenon","level":2,"score":0.28940001130104065},{"id":"https://openalex.org/C21080849","wikidata":"https://www.wikidata.org/wiki/Q13611879","display_name":"Data point","level":2,"score":0.2842000126838684},{"id":"https://openalex.org/C67186912","wikidata":"https://www.wikidata.org/wiki/Q367664","display_name":"Data modeling","level":2,"score":0.2816999852657318},{"id":"https://openalex.org/C160920958","wikidata":"https://www.wikidata.org/wiki/Q7662746","display_name":"Synthetic data","level":2,"score":0.2711000144481659},{"id":"https://openalex.org/C117220453","wikidata":"https://www.wikidata.org/wiki/Q5172842","display_name":"Correlation","level":2,"score":0.2694000005722046},{"id":"https://openalex.org/C55037315","wikidata":"https://www.wikidata.org/wiki/Q5421151","display_name":"Experimental data","level":2,"score":0.2549999952316284},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.25060001015663147}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.20814","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.20814","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.20814","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.20814","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Emergent":[0],"misalignment":[1,18],"(EM)":[2],"is":[3,205],"a":[4],"phenomenon":[5],"in":[6],"which":[7],"models":[8,139],"generalize":[9],"with":[10,81,189,195],"narrow":[11,73,142,150,161],"fine-tuning,":[12,74],"leading":[13],"to":[14,49,61,141],"broad":[15,83],"(yet":[16],"uneven)":[17],"across":[19,53],"evaluation":[20,128,178,185,214],"questions.":[21],"We":[22,41,94],"study":[23],"EM":[24],"and":[25,38,55,100,135,159,163,168,177,184,213],"its":[26],"variability":[27],"directly":[28],"through":[29,67],"the":[30,98,104,115,133,136,171,196],"components":[31],"of":[32,103,114],"fine-tuning:":[33],"training":[34,46,91,176,183],"dynamics,":[35],"model":[36,56,106],"priors,":[37],"data.":[39],"(1)":[40],"first":[42],"explored":[43],"how":[44],"in-domain":[45],"loss":[47],"relates":[48],"out-of-domain":[50],"alignment":[51,84,147],"scores":[52,85,107,148],"datasets":[54],"families.":[57],"Then,":[58],"we":[59,154],"tried":[60],"induce":[62],"potential":[63,121],"alternative":[64],"local":[65],"minima":[66],"different":[68,111],"learning":[69],"schedules":[70],"for":[71,175],"one":[72],"but":[75],"did":[76],"not":[77],"find":[78],"strong":[79],"runs":[80],"better":[82],"conditioned":[86],"on":[87,123],"similar":[88],"or":[89],"lower":[90],"loss.":[92],"(2)":[93],"found":[95,164],"that":[96],"although":[97],"mean":[99],"standard":[101],"deviations":[102],"misaligned":[105],"are":[108,119],"usually":[109],"statistically":[110],"from":[112,131,210],"those":[113],"pre-trained":[116,134],"model,":[117],"there":[118],"some":[120],"signals":[122],"overall":[124],"positive":[125],"correlation.":[126],"The":[127,200],"prompt-only":[129],"activations":[130,187],"both":[132],"original":[137],"instruct":[138],"(prior":[140],"fine-tuning)":[143],"could":[144],"predict":[145],"fine-grained":[146],"after":[149,160],"fine-tuning.":[151],"(3)":[152],"Finally,":[153],"compared":[155],"activation":[156,173],"deltas":[157],"before":[158],"fine-tuning":[162],"moderate-to-high":[165],"subspace":[166],"overlap":[167,204,208],"similarity":[169],"between":[170,182],"resulting":[172],"shifts":[174],"prompts.":[179],"Subspace":[180],"overlaps":[181],"prompt":[186,203],"correlate":[188],"their":[190],"shifts'":[191],"similarities":[192],"when":[193],"measuring":[194],"last":[197],"prompt-token":[198],"activations.":[199,216],"train-evaluation":[201],"data":[202],"controlled":[206],"against":[207],"computed":[209],"random":[211],"vectors":[212],"prompts":[215]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-06-24T00:00:00"}