{"id":"https://openalex.org/W7164414240","doi":"https://doi.org/10.48550/arxiv.2606.12058","title":"Phase Transitions in Attention: A Bayesian Theory of Copy Head Emergence","display_name":"Phase Transitions in Attention: A Bayesian Theory of Copy Head Emergence","publication_year":2026,"publication_date":"2026-06-10","ids":{"openalex":"https://openalex.org/W7164414240","doi":"https://doi.org/10.48550/arxiv.2606.12058"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2606.12058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2606.12058","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5138465384","display_name":"Itay Lavie","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lavie, Itay","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138395730","display_name":"Kirsten Fischer","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Fischer, Kirsten","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5032849147","display_name":"Andrey Lekov","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Lekov, Andrey","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5138412421","display_name":"Frederic Van Maele","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Van Maele, Frederic","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5053390682","display_name":"Zohar Ringel","orcid":"https://orcid.org/0000-0002-2555-7695"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Ringel, Zohar","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5032616213","display_name":"Moritz Helias","orcid":"https://orcid.org/0000-0002-0404-8656"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Helias, Moritz","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.24529999494552612,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},"topics":[{"id":"https://openalex.org/T10465","display_name":"Neurobiology of Language and Bilingualism","score":0.24529999494552612,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T10581","display_name":"Neural dynamics and brain function","score":0.08649999648332596,"subfield":{"id":"https://openalex.org/subfields/2805","display_name":"Cognitive Neuroscience"},"field":{"id":"https://openalex.org/fields/28","display_name":"Neuroscience"},"domain":{"id":"https://openalex.org/domains/1","display_name":"Life Sciences"}},{"id":"https://openalex.org/T12808","display_name":"Ferroelectric and Negative Capacitance Devices","score":0.05649999901652336,"subfield":{"id":"https://openalex.org/subfields/2208","display_name":"Electrical and Electronic Engineering"},"field":{"id":"https://openalex.org/fields/22","display_name":"Engineering"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/softmax-function","display_name":"Softmax function","score":0.8490999937057495},{"id":"https://openalex.org/keywords/bayesian-probability","display_name":"Bayesian probability","score":0.6069999933242798},{"id":"https://openalex.org/keywords/feature","display_name":"Feature (linguistics)","score":0.5296000242233276},{"id":"https://openalex.org/keywords/focus","display_name":"Focus (optics)","score":0.48910000920295715},{"id":"https://openalex.org/keywords/contrast","display_name":"Contrast (vision)","score":0.4690000116825104},{"id":"https://openalex.org/keywords/bayesian-inference","display_name":"Bayesian inference","score":0.4332999885082245},{"id":"https://openalex.org/keywords/sampling","display_name":"Sampling (signal processing)","score":0.3707999885082245},{"id":"https://openalex.org/keywords/layer","display_name":"Layer (electronics)","score":0.3675999939441681},{"id":"https://openalex.org/keywords/pattern-recognition","display_name":"Pattern recognition (psychology)","score":0.3513000011444092}],"concepts":[{"id":"https://openalex.org/C188441871","wikidata":"https://www.wikidata.org/wiki/Q7554146","display_name":"Softmax function","level":3,"score":0.8490999937057495},{"id":"https://openalex.org/C107673813","wikidata":"https://www.wikidata.org/wiki/Q812534","display_name":"Bayesian probability","level":2,"score":0.6069999933242798},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5608000159263611},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5372999906539917},{"id":"https://openalex.org/C2776401178","wikidata":"https://www.wikidata.org/wiki/Q12050496","display_name":"Feature (linguistics)","level":2,"score":0.5296000242233276},{"id":"https://openalex.org/C192209626","wikidata":"https://www.wikidata.org/wiki/Q190909","display_name":"Focus (optics)","level":2,"score":0.48910000920295715},{"id":"https://openalex.org/C2776502983","wikidata":"https://www.wikidata.org/wiki/Q690182","display_name":"Contrast (vision)","level":2,"score":0.4690000116825104},{"id":"https://openalex.org/C160234255","wikidata":"https://www.wikidata.org/wiki/Q812535","display_name":"Bayesian inference","level":3,"score":0.4332999885082245},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.3856000006198883},{"id":"https://openalex.org/C140779682","wikidata":"https://www.wikidata.org/wiki/Q210868","display_name":"Sampling (signal processing)","level":3,"score":0.3707999885082245},{"id":"https://openalex.org/C2779227376","wikidata":"https://www.wikidata.org/wiki/Q6505497","display_name":"Layer (electronics)","level":2,"score":0.3675999939441681},{"id":"https://openalex.org/C153180895","wikidata":"https://www.wikidata.org/wiki/Q7148389","display_name":"Pattern recognition (psychology)","level":2,"score":0.3513000011444092},{"id":"https://openalex.org/C149288129","wikidata":"https://www.wikidata.org/wiki/Q185357","display_name":"Phase transition","level":2,"score":0.34619998931884766},{"id":"https://openalex.org/C33724603","wikidata":"https://www.wikidata.org/wiki/Q812540","display_name":"Bayesian network","level":2,"score":0.3458000123500824},{"id":"https://openalex.org/C44280652","wikidata":"https://www.wikidata.org/wiki/Q104837","display_name":"Phase (matter)","level":2,"score":0.3400000035762787},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.32850000262260437},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.32339999079704285},{"id":"https://openalex.org/C101112237","wikidata":"https://www.wikidata.org/wiki/Q4874481","display_name":"Bayesian statistics","level":4,"score":0.3215999901294708},{"id":"https://openalex.org/C163175372","wikidata":"https://www.wikidata.org/wiki/Q3339222","display_name":"Linear model","level":2,"score":0.3127000033855438},{"id":"https://openalex.org/C2780312720","wikidata":"https://www.wikidata.org/wiki/Q5689100","display_name":"Head (geology)","level":2,"score":0.30959999561309814},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3052000105381012},{"id":"https://openalex.org/C57830394","wikidata":"https://www.wikidata.org/wiki/Q278079","display_name":"Posterior probability","level":3,"score":0.30239999294281006},{"id":"https://openalex.org/C89611455","wikidata":"https://www.wikidata.org/wiki/Q6804646","display_name":"Mechanism (biology)","level":2,"score":0.29350000619888306},{"id":"https://openalex.org/C148483581","wikidata":"https://www.wikidata.org/wiki/Q446488","display_name":"Feature selection","level":2,"score":0.28760001063346863},{"id":"https://openalex.org/C194232998","wikidata":"https://www.wikidata.org/wiki/Q1606712","display_name":"Transition (genetics)","level":3,"score":0.28200000524520874},{"id":"https://openalex.org/C207201462","wikidata":"https://www.wikidata.org/wiki/Q182505","display_name":"Bayes' theorem","level":3,"score":0.2793000042438507},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.2741999924182892},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.2619999945163727}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2606.12058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2606.12058","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2606.12058","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Attention":[0],"is":[1,48,130],"the":[2,37,41,68,87,138,151,155,160],"key":[3],"mechanism":[4],"underlying":[5],"in-context":[6],"learning":[7,29],"in":[8,30,40,86,122,163],"transformers,":[9],"and":[10,71,99,111],"attention":[11,55,69,110,115,124,140],"patterns":[12],"have":[13],"been":[14],"observed":[15,162],"empirically":[16],"to":[17,74],"emerge":[18],"abruptly":[19],"during":[20],"training.":[21],"We":[22,62,104],"present":[23],"a":[24,52,59,64,75,83,117,133,146],"Bayesian":[25,97],"theory":[26],"of":[27,44,89,150,154,159],"feature":[28],"attention;":[31],"we":[32,93],"then":[33],"focus":[34],"on":[35,58],"how":[36],"copy":[38,60,156],"subcircuit":[39],"first":[42],"layer":[43],"an":[45,125],"induction":[46],"head":[47],"learned":[49],"by":[50,132],"analyzing":[51],"single-layer":[53],"softmax":[54,114],"network":[56],"trained":[57],"task.":[61],"derive":[63],"closed-form":[65],"posterior":[66],"over":[67],"matrix":[70],"reduce":[72],"it":[73],"low-dimensional":[76],"order":[77],"parameter":[78],"space.":[79],"This":[80],"reduction":[81],"reveals":[82],"phase":[84,119,128],"transition":[85],"amount":[88],"training":[90,101,164],"data,":[91],"which":[92],"verify":[94],"using":[95],"both":[96],"sampling":[98],"standard":[100],"with":[102,108],"Adam.":[103],"contrast":[105],"our":[106],"results":[107],"linear":[109,123],"find":[112],"that":[113],"exhibits":[116],"\\emph{first-order":[118],"transition}":[120,129],"while":[121],"initial":[126],"\\emph{second-order":[127],"followed":[131],"smooth,":[134],"continuous":[135],"evolution":[136],"toward":[137],"structured":[139],"pattern":[141],"(\\emph{crossover}).":[142],"Our":[143],"work":[144],"provides":[145],"first-principles":[147],"theoretical":[148],"account":[149],"abrupt":[152],"emergence":[153],"subcircuit,":[157],"reminiscent":[158],"one":[161],"large":[165],"language":[166],"models.":[167]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2026-06-12T00:00:00"}