{"id":"https://openalex.org/W4415313672","doi":"https://doi.org/10.48550/arxiv.2510.08341","title":"Post-Norm can Resharpen Attention","display_name":"Post-Norm can Resharpen Attention","publication_year":2025,"publication_date":"2025-10-09","ids":{"openalex":"https://openalex.org/W4415313672","doi":"https://doi.org/10.48550/arxiv.2510.08341"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2510.08341","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.08341","pdf_url":"https://arxiv.org/pdf/2510.08341","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"volume":null,"type":"preprint","indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2510.08341","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5088265340","display_name":"P\u00e1l Zs\u00e1mboki","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I127817813","display_name":"HUN-REN Alfr\u00e9d R\u00e9nyi Institute of Mathematics","ror":"https://ror.org/03vw74f64","country_code":"HU","type":"facility","lineage":["https://openalex.org/I127817813","https://openalex.org/I7597260"]},{"id":"https://openalex.org/I4387152226","display_name":"Hungarian Research Network","ror":"https://ror.org/04w6pnc49","country_code":"HU","type":"other","lineage":["https://openalex.org/I4387152226"]}],"countries":["HU"],"is_corresponding":false,"raw_author_name":"Zs\u00e1mboki, P\u00e1l","raw_affiliation_strings":["HUN-REN Alfr\u00e9d R\u00e9nyi Institute of Mathematics"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"HUN-REN Alfr\u00e9d R\u00e9nyi Institute of Mathematics","institution_ids":["https://openalex.org/I127817813","https://openalex.org/I4387152226"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Levi, Benjamin","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I5388228","display_name":"University of Rochester","ror":"https://ror.org/022kthw22","country_code":"US","type":"education","lineage":["https://openalex.org/I5388228"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Levi, Benjamin","raw_affiliation_strings":["University of Rochester"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Rochester","institution_ids":["https://openalex.org/I5388228"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5075965166","display_name":"David Cecil Smith","orcid":"https://orcid.org/0000-0002-7860-9452","observed_orcids":["https://orcid.org/0000-0002-7860-9452","https://orcid.org/0000-0003-1886-8193","https://orcid.org/0009-0002-6137-6741"]},"institutions":[{"id":"https://openalex.org/I17301866","display_name":"University of Alabama","ror":"https://ror.org/03xrrjk67","country_code":"US","type":"education","lineage":["https://openalex.org/I17301866"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Smith, David Ansel Josef","raw_affiliation_strings":["The University of Alabama"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"The University of Alabama","institution_ids":["https://openalex.org/I17301866"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5120048844","display_name":"Mitansh Kagalwala","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I51556381","display_name":"University of Virginia","ror":"https://ror.org/0153tk833","country_code":"US","type":"education","lineage":["https://openalex.org/I51556381"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kagalwala, Mitansh","raw_affiliation_strings":["University of Virginia , Charlottesville"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Virginia , Charlottesville","institution_ids":["https://openalex.org/I51556381"]}]},{"author_position":"middle","author":{"id":null,"display_name":"Kell, Arlington","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I130701444","display_name":"Georgia Institute of Technology","ror":"https://ror.org/01zkghx44","country_code":"US","type":"education","lineage":["https://openalex.org/I130701444"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Kell, Arlington","raw_affiliation_strings":["Georgia Institute of Technology"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Georgia Institute of Technology","institution_ids":["https://openalex.org/I130701444"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5008683416","display_name":"Samuel Liechty","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I100005738","display_name":"Brigham Young University","ror":"https://ror.org/047rhhm47","country_code":"US","type":"education","lineage":["https://openalex.org/I100005738"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Liechty, Samuel","raw_affiliation_strings":["Brigham Young University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Brigham Young University","institution_ids":["https://openalex.org/I100005738"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5059484322","display_name":"Cong Wang","orcid":"https://orcid.org/0000-0002-3771-9954","observed_orcids":["https://orcid.org/0000-0002-3771-9954"]},"institutions":[{"id":"https://openalex.org/I188497080","display_name":"Carleton College","ror":"https://ror.org/03jep7677","country_code":"US","type":"education","lineage":["https://openalex.org/I188497080"]}],"countries":["US"],"is_corresponding":false,"raw_author_name":"Wang, Cong","raw_affiliation_strings":["Carleton College"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Carleton College","institution_ids":["https://openalex.org/I188497080"]}]}],"institutions":[],"countries_distinct_count":2,"institutions_distinct_count":8,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7343999743461609,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.7343999743461609,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10181","display_name":"Natural Language Processing Techniques","score":0.1421000063419342,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11567","display_name":"semigroups and automata theory","score":0.01679999940097332,"subfield":{"id":"https://openalex.org/subfields/1703","display_name":"Computational Theory and Mathematics"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/length-generalization","display_name":"length generalization","score":0.9919999837875366},{"id":"https://openalex.org/keywords/attention-dispersion","display_name":"attention dispersion","score":0.5809999704360962},{"id":"https://openalex.org/keywords/next-token-prediction","display_name":"next-token prediction","score":0.625},{"id":"https://openalex.org/keywords/transformer","display_name":"transformer models","score":0.44200000166893005},{"id":"https://openalex.org/keywords/exponentially-weighted-moving-average","display_name":"exponential moving average","score":0.7319999933242798},{"id":"https://openalex.org/keywords/noisy-gradients","display_name":"noisy gradients","score":0.8650000095367432},{"id":"https://openalex.org/keywords/algorithmic-tasks","display_name":"algorithmic tasks","score":0.4309999942779541},{"id":"https://openalex.org/keywords/benchmark","display_name":"benchmark","score":0.42800000309944153}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.743399977684021},{"id":"https://openalex.org/C112313634","wikidata":"https://www.wikidata.org/wiki/Q7886648","display_name":"Complement (music)","level":5,"score":0.6430000066757202},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.5827999711036682},{"id":"https://openalex.org/C177148314","wikidata":"https://www.wikidata.org/wiki/Q170084","display_name":"Generalization","level":2,"score":0.5648000240325928},{"id":"https://openalex.org/C79581498","wikidata":"https://www.wikidata.org/wiki/Q1367530","display_name":"Suite","level":2,"score":0.4869999885559082},{"id":"https://openalex.org/C80444323","wikidata":"https://www.wikidata.org/wiki/Q2878974","display_name":"Theoretical computer science","level":1,"score":0.45809999108314514},{"id":"https://openalex.org/C177264268","wikidata":"https://www.wikidata.org/wiki/Q1514741","display_name":"Set (abstract data type)","level":2,"score":0.4359999895095825},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.42730000615119934},{"id":"https://openalex.org/C43126263","wikidata":"https://www.wikidata.org/wiki/Q128751","display_name":"Source code","level":2,"score":0.38519999384880066},{"id":"https://openalex.org/C2780586882","wikidata":"https://www.wikidata.org/wiki/Q7520643","display_name":"Simple (philosophy)","level":2,"score":0.3790999948978424},{"id":"https://openalex.org/C185798385","wikidata":"https://www.wikidata.org/wiki/Q1161707","display_name":"Benchmark (surveying)","level":2,"score":0.3628000020980835},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.35530000925064087},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.35100001096725464},{"id":"https://openalex.org/C554936623","wikidata":"https://www.wikidata.org/wiki/Q199657","display_name":"Reading (process)","level":2,"score":0.349700003862381},{"id":"https://openalex.org/C151376022","wikidata":"https://www.wikidata.org/wiki/Q168698","display_name":"Exponential function","level":2,"score":0.34769999980926514},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.34619998931884766},{"id":"https://openalex.org/C137293760","wikidata":"https://www.wikidata.org/wiki/Q3621696","display_name":"Language model","level":2,"score":0.32350000739097595},{"id":"https://openalex.org/C129844170","wikidata":"https://www.wikidata.org/wiki/Q41299","display_name":"Quadratic equation","level":2,"score":0.3057999908924103},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.28119999170303345},{"id":"https://openalex.org/C149441793","wikidata":"https://www.wikidata.org/wiki/Q200726","display_name":"Probability distribution","level":2,"score":0.2766999900341034},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.2703000009059906},{"id":"https://openalex.org/C151552104","wikidata":"https://www.wikidata.org/wiki/Q7705809","display_name":"Test suite","level":4,"score":0.25999999046325684},{"id":"https://openalex.org/C198352243","wikidata":"https://www.wikidata.org/wiki/Q37105","display_name":"Line (geometry)","level":2,"score":0.2599000036716461},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.2556999921798706}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2510.08341","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.08341","pdf_url":"https://arxiv.org/pdf/2510.08341","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2510.08341","doi":"https://doi.org/10.48550/arxiv.2510.08341","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2510.08341","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2510.08341","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2510.08341","pdf_url":"https://arxiv.org/pdf/2510.08341","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"sustainable_development_goals_aurora":[],"study_designs":[],"awards":[{"id":"https://openalex.org/G8082130625","display_name":null,"funder_award_id":"RRF-2.3.1-21-2022-00004","funder_id":"https://openalex.org/F4320326762","funder_display_name":"Nemzeti Kutat\u00e1si Fejleszt\u00e9si \u00e9s Innov\u00e1ci\u00f3s Hivatal"}],"funders":[{"id":"https://openalex.org/F4320326762","display_name":"Nemzeti Kutat\u00e1si Fejleszt\u00e9si \u00e9s Innov\u00e1ci\u00f3s Hivatal","ror":null},{"id":"https://openalex.org/F4320336675","display_name":"National Research, Development and Innovation Office","ror":null}],"has_content":{"pdf":true,"grobid_xml":true},"content_urls":{"pdf":"https://content.openalex.org/works/W4415313672.pdf","grobid_xml":"https://content.openalex.org/works/W4415313672.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Length":[0],"Generalization":[1],"is":[2,40],"the":[3,32,45,65,69,81,139,154],"essential":[4],"capacity":[5],"of":[6,48,62,108,141,157,164],"autonomous":[7],"agents":[8],"to":[9,41,72,102,126],"perform":[10],"tasks":[11],"in":[12,36,64,80],"longer":[13],"contexts":[14],"than":[15],"those":[16],"encountered":[17],"during":[18],"training.":[19],"To":[20],"systematically":[21],"study":[22],"this":[23,60,95,128],"feat,":[24],"we":[25],"test":[26],"how":[27,109],"well":[28],"models":[29],"can":[30,91,119,137],"approximate":[31],"next":[33,50,148],"token":[34],"distributions":[35],"algorithmic":[37],"tasks.":[38],"This":[39],"take":[42],"into":[43],"account":[44],"realistic":[46],"possibility":[47],"multiple":[49],"tokens":[51,78,149],"being":[52],"legal.":[53,151],"We":[54,83,122,130,152],"present":[55,123],"a":[56,74,85,116,162],"prototypical":[57],"benchmark":[58],"for":[59],"line":[61],"study:":[63],"Set":[66],"Complement":[67],"Task,":[68],"model":[70],"needs":[71],"output":[73],"uniform":[75],"distribution":[76],"over":[77],"not":[79],"input.":[82],"prove":[84],"theorem":[86],"that":[87,133,144],"states":[88],"simple":[89],"transformers":[90],"length":[92],"generalize":[93],"on":[94,161],"task,":[96],"however,":[97],"with":[98],"performance":[99],"degradation":[100],"due":[101],"attention":[103],"dispersion.":[104],"A":[105],"mechanistic":[106],"reading":[107],"dispersion":[110],"takes":[111],"effect":[112],"lets":[113],"us":[114],"discover":[115],"remedy:":[117],"Post-Norm":[118],"Resharpen":[120],"Attention.":[121],"experimental":[124],"evidence":[125],"support":[127],"idea.":[129],"also":[131],"show":[132],"Exponential":[134],"Moving":[135],"Averages":[136],"help":[138],"issue":[140],"noisy":[142],"gradients":[143],"arises":[145],"when":[146],"many":[147],"are":[150],"validate":[153],"general":[155],"applicability":[156],"our":[158],"proposed":[159],"methods":[160],"suite":[163],"formal":[165],"language":[166],"experiments.":[167],"Our":[168],"source":[169],"code":[170],"will":[171],"be":[172],"available":[173],"upon":[174],"publication.":[175]},"counts_by_year":[],"updated_date":"2026-10-16T09:59:22.235173","created_date":"2025-10-18T00:00:00"}