{"id":"https://openalex.org/W7161849683","doi":"https://doi.org/10.48550/arxiv.2605.18870","title":"Multi-Headed Transformer Architectures as Time-dependent Wasserstein Gradient Flows","display_name":"Multi-Headed Transformer Architectures as Time-dependent Wasserstein Gradient Flows","publication_year":2026,"publication_date":"2026-05-15","ids":{"openalex":"https://openalex.org/W7161849683","doi":"https://doi.org/10.48550/arxiv.2605.18870"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.18870","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18870","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.18870","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5099282196","display_name":"Alex Massucco","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Massucco, Alex","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5093271776","display_name":"Leonardo Del Grande","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Del Grande, Leonardo","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136515769","display_name":"Marcello Carioni","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Carioni, Marcello","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136569775","display_name":"Christoff Brune","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Brune, Christoph","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136575717","display_name":"Carola-Bibiane Sch\u00f6nlieb","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sch\u00f6nlieb, Carola-Bibiane","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.24369999766349792,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11206","display_name":"Model Reduction and Neural Networks","score":0.24369999766349792,"subfield":{"id":"https://openalex.org/subfields/3109","display_name":"Statistical and Nonlinear Physics"},"field":{"id":"https://openalex.org/fields/31","display_name":"Physics and Astronomy"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10229","display_name":"Geometric Analysis and Curvature Flows","score":0.08510000258684158,"subfield":{"id":"https://openalex.org/subfields/2604","display_name":"Applied Mathematics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11205","display_name":"Numerical methods in inverse problems","score":0.04390000179409981,"subfield":{"id":"https://openalex.org/subfields/2610","display_name":"Mathematical Physics"},"field":{"id":"https://openalex.org/fields/26","display_name":"Mathematics"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/balanced-flow","display_name":"Balanced flow","score":0.8058000206947327},{"id":"https://openalex.org/keywords/initialization","display_name":"Initialization","score":0.6430000066757202},{"id":"https://openalex.org/keywords/uniqueness","display_name":"Uniqueness","score":0.5568000078201294},{"id":"https://openalex.org/keywords/perturbation","display_name":"Perturbation (astronomy)","score":0.4300000071525574},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.38100001215934753},{"id":"https://openalex.org/keywords/gradient-method","display_name":"Gradient method","score":0.373199999332428},{"id":"https://openalex.org/keywords/wasserstein-metric","display_name":"Wasserstein metric","score":0.3702999949455261},{"id":"https://openalex.org/keywords/robustness","display_name":"Robustness (evolution)","score":0.36880001425743103}],"concepts":[{"id":"https://openalex.org/C167879884","wikidata":"https://www.wikidata.org/wiki/Q727568","display_name":"Balanced flow","level":2,"score":0.8058000206947327},{"id":"https://openalex.org/C114466953","wikidata":"https://www.wikidata.org/wiki/Q6034165","display_name":"Initialization","level":2,"score":0.6430000066757202},{"id":"https://openalex.org/C2777021972","wikidata":"https://www.wikidata.org/wiki/Q22976830","display_name":"Uniqueness","level":2,"score":0.5568000078201294},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.47909998893737793},{"id":"https://openalex.org/C177918212","wikidata":"https://www.wikidata.org/wiki/Q803623","display_name":"Perturbation (astronomy)","level":2,"score":0.4300000071525574},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.38100001215934753},{"id":"https://openalex.org/C115680565","wikidata":"https://www.wikidata.org/wiki/Q5977448","display_name":"Gradient method","level":2,"score":0.373199999332428},{"id":"https://openalex.org/C2777634741","wikidata":"https://www.wikidata.org/wiki/Q768993","display_name":"Wasserstein metric","level":2,"score":0.3702999949455261},{"id":"https://openalex.org/C63479239","wikidata":"https://www.wikidata.org/wiki/Q7353546","display_name":"Robustness (evolution)","level":3,"score":0.36880001425743103},{"id":"https://openalex.org/C188198153","wikidata":"https://www.wikidata.org/wiki/Q1613840","display_name":"Limiting","level":2,"score":0.3668999969959259},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3571999967098236},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.35510000586509705},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3377000093460083},{"id":"https://openalex.org/C61445026","wikidata":"https://www.wikidata.org/wiki/Q217608","display_name":"Fixed point","level":2,"score":0.3303000032901764},{"id":"https://openalex.org/C2777303404","wikidata":"https://www.wikidata.org/wiki/Q759757","display_name":"Convergence (economics)","level":2,"score":0.33000001311302185},{"id":"https://openalex.org/C206815995","wikidata":"https://www.wikidata.org/wiki/Q1341307","display_name":"Energy flow","level":3,"score":0.32179999351501465},{"id":"https://openalex.org/C112972136","wikidata":"https://www.wikidata.org/wiki/Q7595718","display_name":"Stability (learning theory)","level":2,"score":0.319599986076355},{"id":"https://openalex.org/C134306372","wikidata":"https://www.wikidata.org/wiki/Q7754","display_name":"Mathematical analysis","level":1,"score":0.3111000061035156},{"id":"https://openalex.org/C38349280","wikidata":"https://www.wikidata.org/wiki/Q1434290","display_name":"Flow (mathematics)","level":2,"score":0.30970001220703125},{"id":"https://openalex.org/C121864883","wikidata":"https://www.wikidata.org/wiki/Q677916","display_name":"Statistical physics","level":1,"score":0.3091000020503998},{"id":"https://openalex.org/C187691185","wikidata":"https://www.wikidata.org/wiki/Q2020720","display_name":"Grid","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.29660001397132874},{"id":"https://openalex.org/C91188154","wikidata":"https://www.wikidata.org/wiki/Q186247","display_name":"Vector field","level":2,"score":0.27730000019073486},{"id":"https://openalex.org/C186370098","wikidata":"https://www.wikidata.org/wiki/Q442787","display_name":"Energy (signal processing)","level":2,"score":0.2615000009536743}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.18870","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18870","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.18870","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.18870","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[{"display_name":"Affordable and clean energy","score":0.44394922256469727,"id":"https://metadata.un.org/sdg/7"}],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"In":[0,47],"recent":[1],"years,":[2],"transformer":[3,61],"architectures":[4,40,62],"have":[5],"revolutionized":[6],"the":[7,13,26,31,38,56,73,76,101,114,117,122,126,134,145,148,155,159,163,168,171,184,188,194,197,202,205,210,215,218,232,238,242,246,250],"field":[8],"of":[9,24,75,116,121,125,133,147,153,170,183,193,204,217,241],"language":[10],"processing,":[11],"opening":[12],"door":[14],"to":[15,86,176,209,214],"previously":[16],"unforeseen":[17],"possibilities.":[18],"However,":[19],"from":[20],"a":[21,68,109,130,138,180],"theoretical":[22,225],"point":[23,132],"view,":[25],"mathematical":[27],"models":[28,173],"proposed":[29,172],"in":[30,59,244],"literature":[32],"often":[33],"lack":[34],"direct":[35],"contact":[36],"with":[37,174,227],"actual":[39],"and":[41,93,158,191,236,249],"depend":[42],"on":[43,82,100,113,162,187],"strong":[44],"simplifying":[45],"assumptions.":[46],"this":[48,52],"paper,":[49],"we":[50,105,143,166,200],"reduce":[51],"gap":[53],"by":[54],"modelling":[55],"data":[57,157,190],"flow":[58],"multi-headed":[60],"as":[63],"time-dependent":[64],"gradient":[65,127,149,185,220],"flows":[66,128,150,186],"for":[67,90,94],"suitable":[69,110],"interaction":[70,135,207],"energy":[71,136,208],"capturing":[72],"design":[74],"attention":[77],"mechanism.":[78],"The":[79],"explicit":[80],"dependence":[81,182],"time":[83],"allows":[84],"us":[85],"consider":[87],"different":[88],"weights":[89],"each":[91,95,119],"head":[92],"layer,":[96],"without":[97],"imposing":[98],"constraints":[99],"initialization":[102],"method.":[103],"Moreover,":[104],"prove":[106,201],"that,":[107],"under":[108],"integrability":[111],"assumption":[112],"evolution":[115],"weights,":[118],"element":[120],"$\u03c9$-limit":[123],"set":[124],"is":[129],"stationary":[131],"at":[137],"limiting":[139],"weight":[140],"distribution.":[141],"Finally,":[142],"analyse":[144],"stability":[146],"considering":[151],"perturbations":[152],"both":[154,245],"initial":[156,189],"weights.":[160],"Specifically,":[161],"one":[164],"hand,":[165,199],"study":[167],"robustness":[169],"respect":[175],"noisy":[177],"inputs,":[178],"establishing":[179],"continuous":[181],"uniqueness":[192],"flows.":[195,221],"On":[196],"other":[198],"$\u0393$-convergence":[203],"perturbed":[206],"unperturbed":[211],"one,":[212],"leading":[213],"convergence":[216],"corresponding":[219],"We":[222],"complement":[223],"these":[224],"results":[226],"numerical":[228],"experiments":[229],"that":[230],"confirm":[231],"predicted":[233],"energy-dissipation":[234],"identity":[235],"clarify":[237],"asymptotic":[239],"behavior":[240],"dynamics":[243],"autonomous-like":[247],"(Ornstein--Uhlenbeck)":[248],"genuinely":[251],"non-autonomous":[252],"(oscillating-weights)":[253],"regimes.":[254]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-21T00:00:00"}