{"id":"https://openalex.org/W7077041677","doi":"https://doi.org/10.48550/arxiv.2508.03616","title":"Hidden Dynamics of Massive Activations in Transformer Training","display_name":"Hidden Dynamics of Massive Activations in Transformer Training","publication_year":2025,"publication_date":"2025-08-05","ids":{"openalex":"https://openalex.org/W7077041677","doi":"https://doi.org/10.48550/arxiv.2508.03616"},"language":"en","primary_location":{"id":"doi:10.48550/arxiv.2508.03616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.03616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2508.03616","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":null,"display_name":"Gallego-Feliciano, Jorge","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Gallego-Feliciano, Jorge","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"McClendon, S. Aaron","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"McClendon, S. Aaron","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Morinelli, Juan","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Morinelli, Juan","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":null,"display_name":"Zervoudakis, Stavros","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Zervoudakis, Stavros","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":null,"display_name":"Saravanos, Antonios","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Saravanos, Antonios","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":true,"primary_topic":{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.6700000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T12157","display_name":"Geochemistry and Geologic Mapping","score":0.6700000166893005,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T13067","display_name":"Geological Modeling and Analysis","score":0.029400000348687172,"subfield":{"id":"https://openalex.org/subfields/1906","display_name":"Geochemistry and Petrology"},"field":{"id":"https://openalex.org/fields/19","display_name":"Earth and Planetary Sciences"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T14311","display_name":"Electrical and Electromagnetic Research","score":0.01979999989271164,"subfield":{"id":"https://openalex.org/subfields/3107","display_name":"Atomic and Molecular Physics, and Optics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.6561999917030334},{"id":"https://openalex.org/keywords/key","display_name":"Key (lock)","score":0.5335999727249146},{"id":"https://openalex.org/keywords/code","display_name":"Code (set theory)","score":0.3928999900817871},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.375},{"id":"https://openalex.org/keywords/function","display_name":"Function (biology)","score":0.3734000027179718},{"id":"https://openalex.org/keywords/logarithm","display_name":"Logarithm","score":0.32089999318122864},{"id":"https://openalex.org/keywords/control","display_name":"Control (management)","score":0.3057999908924103}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7110999822616577},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.6561999917030334},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5335999727249146},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.4341000020503998},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42080000042915344},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.3928999900817871},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.375},{"id":"https://openalex.org/C14036430","wikidata":"https://www.wikidata.org/wiki/Q3736076","display_name":"Function (biology)","level":2,"score":0.3734000027179718},{"id":"https://openalex.org/C39927690","wikidata":"https://www.wikidata.org/wiki/Q11197","display_name":"Logarithm","level":2,"score":0.32089999318122864},{"id":"https://openalex.org/C2775924081","wikidata":"https://www.wikidata.org/wiki/Q55608371","display_name":"Control (management)","level":2,"score":0.3057999908924103},{"id":"https://openalex.org/C76969082","wikidata":"https://www.wikidata.org/wiki/Q486902","display_name":"Mathematical model","level":2,"score":0.2985999882221222},{"id":"https://openalex.org/C145912823","wikidata":"https://www.wikidata.org/wiki/Q113558","display_name":"Dynamics (music)","level":2,"score":0.29679998755455017},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.2888999879360199},{"id":"https://openalex.org/C77405623","wikidata":"https://www.wikidata.org/wiki/Q598451","display_name":"System dynamics","level":2,"score":0.2793000042438507},{"id":"https://openalex.org/C195672273","wikidata":"https://www.wikidata.org/wiki/Q6888132","display_name":"Model-based design","level":2,"score":0.2669000029563904},{"id":"https://openalex.org/C133731056","wikidata":"https://www.wikidata.org/wiki/Q4917288","display_name":"Control engineering","level":1,"score":0.260699987411499},{"id":"https://openalex.org/C44154836","wikidata":"https://www.wikidata.org/wiki/Q45045","display_name":"Simulation","level":1,"score":0.25380000472068787},{"id":"https://openalex.org/C2777904410","wikidata":"https://www.wikidata.org/wiki/Q7397","display_name":"Software","level":2,"score":0.2535000145435333}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2508.03616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.03616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2508.03616","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2508.03616","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":false,"pdf":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"We":[0,68],"present":[1],"the":[2,14,131],"first":[3],"comprehensive":[4],"analysis":[5,33],"of":[6,34,108,133],"massive":[7,45,109,134],"activation":[8,46,110],"development":[9],"throughout":[10],"transformer":[11],"training,":[12],"using":[13,58],"Pythia":[15],"model":[16,36,119,139],"family":[17],"as":[18],"our":[19,23],"testbed,":[20],"and":[21,89,95,103,125,141,145],"release":[22],"full":[24],"dataset":[25],"publicly":[26],"to":[27,74,101],"support":[28],"further":[29],"research.":[30],"Through":[31],"systematic":[32],"various":[35],"sizes":[37],"across":[38],"multiple":[39],"training":[40,121,149],"checkpoints,":[41],"we":[42],"demonstrate":[43,129],"that":[44,53,130],"emergence":[47,93,111,132],"follows":[48],"highly":[49],"predictable":[50],"mathematical":[51,77],"patterns":[52],"can":[54,142],"be":[55,143],"accurately":[56],"modeled":[57],"an":[59],"exponentially-modulated":[60],"logarithmic":[61],"function":[62],"with":[63,115],"five":[64],"key":[65,106],"parameters.":[66],"Additionally,":[67],"develop":[69],"a":[70],"machine":[71],"learning":[72],"framework":[73],"predict":[75,102],"these":[76],"parameters":[78],"from":[79],"architectural":[80],"specifications":[81],"alone,":[82],"achieving":[83],"high":[84],"accuracy":[85,91],"for":[86,92,118],"steady-state":[87],"behavior":[88],"moderate":[90],"timing":[94],"magnitude.":[96],"These":[97],"findings":[98,128],"enable":[99],"architects":[100],"potentially":[104,146],"control":[105],"aspects":[107],"through":[112],"design":[113,140],"choices,":[114],"significant":[116],"implications":[117],"stability,":[120],"cycle":[122],"length,":[123],"interpretability,":[124],"optimization.":[126],"Our":[127],"activations":[135],"is":[136,152],"governed":[137],"by":[138],"anticipated,":[144],"controlled,":[147],"before":[148],"begins.":[150],"Code":[151],"available":[153],"at":[154],"https://github.com/Aimpoint-Digital/massive-activations-fork":[155]},"counts_by_year":[],"updated_date":"2026-07-28T07:46:37.118299","created_date":"2025-10-10T00:00:00"}