{"id":"https://openalex.org/W7161687274","doi":"https://doi.org/10.48550/arxiv.2605.17660","title":"Training Infinitely Deep and Wide Transformers","display_name":"Training Infinitely Deep and Wide Transformers","publication_year":2026,"publication_date":"2026-05-17","ids":{"openalex":"https://openalex.org/W7161687274","doi":"https://doi.org/10.48550/arxiv.2605.17660"},"language":null,"primary_location":{"id":"doi:10.48550/arxiv.2605.17660","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17660","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://doi.org/10.48550/arxiv.2605.17660","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5076373842","display_name":"Rapha\u00ebl Barboni","orcid":"https://orcid.org/0000-0002-5323-7939"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Barboni, Rapha\u00ebl","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5026803226","display_name":"Maarten V. de Hoop","orcid":"https://orcid.org/0000-0002-6333-0379"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"de Hoop, Maarten V.","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5136471788","display_name":"Takashi Furuya","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Furuya, Takashi","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5136484492","display_name":"Gabriel Peyr\u00e9","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Peyr\u00e9, Gabriel","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":false,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.79339998960495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.79339998960495,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10775","display_name":"Generative Adversarial Networks and Image Synthesis","score":0.09480000287294388,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.020999999716877937,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/maxima-and-minima","display_name":"Maxima and minima","score":0.6775000095367432},{"id":"https://openalex.org/keywords/affine-transformation","display_name":"Affine transformation","score":0.5570999979972839},{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.5134000182151794},{"id":"https://openalex.org/keywords/gradient-descent","display_name":"Gradient descent","score":0.48579999804496765},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.4577000141143799},{"id":"https://openalex.org/keywords/spurious-relationship","display_name":"Spurious relationship","score":0.41920000314712524},{"id":"https://openalex.org/keywords/gramian-matrix","display_name":"Gramian matrix","score":0.413100004196167},{"id":"https://openalex.org/keywords/jacobian-matrix-and-determinant","display_name":"Jacobian matrix and determinant","score":0.38760000467300415},{"id":"https://openalex.org/keywords/uniqueness","display_name":"Uniqueness","score":0.3619000017642975}],"concepts":[{"id":"https://openalex.org/C186633575","wikidata":"https://www.wikidata.org/wiki/Q845060","display_name":"Maxima and minima","level":2,"score":0.6775000095367432},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.5608000159263611},{"id":"https://openalex.org/C92757383","wikidata":"https://www.wikidata.org/wiki/Q382497","display_name":"Affine transformation","level":2,"score":0.5570999979972839},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.5134000182151794},{"id":"https://openalex.org/C153258448","wikidata":"https://www.wikidata.org/wiki/Q1199743","display_name":"Gradient descent","level":3,"score":0.48579999804496765},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.4577000141143799},{"id":"https://openalex.org/C97256817","wikidata":"https://www.wikidata.org/wiki/Q1462316","display_name":"Spurious relationship","level":2,"score":0.41920000314712524},{"id":"https://openalex.org/C77246614","wikidata":"https://www.wikidata.org/wiki/Q1409400","display_name":"Gramian matrix","level":3,"score":0.413100004196167},{"id":"https://openalex.org/C28826006","wikidata":"https://www.wikidata.org/wiki/Q33521","display_name":"Applied mathematics","level":1,"score":0.3939000070095062},{"id":"https://openalex.org/C200331156","wikidata":"https://www.wikidata.org/wiki/Q506041","display_name":"Jacobian matrix and determinant","level":2,"score":0.38760000467300415},{"id":"https://openalex.org/C2777021972","wikidata":"https://www.wikidata.org/wiki/Q22976830","display_name":"Uniqueness","level":2,"score":0.3619000017642975},{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.3517000079154968},{"id":"https://openalex.org/C163716315","wikidata":"https://www.wikidata.org/wiki/Q901177","display_name":"Gaussian","level":2,"score":0.3488999903202057},{"id":"https://openalex.org/C138187205","wikidata":"https://www.wikidata.org/wiki/Q131251","display_name":"Tangent","level":2,"score":0.3440999984741211},{"id":"https://openalex.org/C11413529","wikidata":"https://www.wikidata.org/wiki/Q8366","display_name":"Algorithm","level":1,"score":0.3416999876499176},{"id":"https://openalex.org/C47446073","wikidata":"https://www.wikidata.org/wiki/Q5165890","display_name":"Control theory (sociology)","level":3,"score":0.3407999873161316},{"id":"https://openalex.org/C167879884","wikidata":"https://www.wikidata.org/wiki/Q727568","display_name":"Balanced flow","level":2,"score":0.33059999346733093},{"id":"https://openalex.org/C54732982","wikidata":"https://www.wikidata.org/wiki/Q1415345","display_name":"Modulo","level":2,"score":0.3255999982357025},{"id":"https://openalex.org/C48145219","wikidata":"https://www.wikidata.org/wiki/Q1335365","display_name":"Security token","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C184720557","wikidata":"https://www.wikidata.org/wiki/Q7825049","display_name":"Topology (electrical circuits)","level":2,"score":0.3208000063896179},{"id":"https://openalex.org/C126255220","wikidata":"https://www.wikidata.org/wiki/Q141495","display_name":"Mathematical optimization","level":1,"score":0.3100000023841858},{"id":"https://openalex.org/C28855332","wikidata":"https://www.wikidata.org/wiki/Q198099","display_name":"Quantization (signal processing)","level":2,"score":0.30399999022483826},{"id":"https://openalex.org/C2777634741","wikidata":"https://www.wikidata.org/wiki/Q768993","display_name":"Wasserstein metric","level":2,"score":0.30300000309944153},{"id":"https://openalex.org/C70153297","wikidata":"https://www.wikidata.org/wiki/Q5591907","display_name":"Gradient boosting","level":3,"score":0.29739999771118164},{"id":"https://openalex.org/C84507854","wikidata":"https://www.wikidata.org/wiki/Q952222","display_name":"Diagonally dominant matrix","level":3,"score":0.2768999934196472},{"id":"https://openalex.org/C176217482","wikidata":"https://www.wikidata.org/wiki/Q860554","display_name":"Metric (unit)","level":2,"score":0.26829999685287476},{"id":"https://openalex.org/C141513077","wikidata":"https://www.wikidata.org/wiki/Q378542","display_name":"Independent and identically distributed random variables","level":3,"score":0.2578999996185303},{"id":"https://openalex.org/C206688291","wikidata":"https://www.wikidata.org/wiki/Q7617819","display_name":"Stochastic gradient descent","level":3,"score":0.25029999017715454}],"mesh":[],"locations_count":1,"locations":[{"id":"doi:10.48550/arxiv.2605.17660","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17660","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"doi:10.48550/arxiv.2605.17660","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.17660","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":null,"issn":null,"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"pdf":false,"grobid_xml":false},"content_urls":null,"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Transformers":[0],"have":[1],"become":[2],"the":[3,11,34,39,74,81,108,137,142,154,163,189,243,254],"dominant":[4],"architecture":[5],"in":[6,33,124,162],"modern":[7],"machine":[8],"learning,":[9],"yet":[10],"theoretical":[12],"understanding":[13],"of":[14,31,42,47,76,90,107,141,158,188,207],"their":[15],"training":[16,30,55,65,143],"dynamics":[17],"remains":[18],"limited.":[19],"This":[20],"paper":[21],"develops":[22],"a":[23,61,69,169,213],"rigorous":[24,170],"mathematical":[25],"framework":[26],"for":[27,136,172,186,194],"analyzing":[28],"gradient-based":[29,173],"transformers":[32],"mean-field":[35,85],"regime,":[36],"where":[37],"both":[38],"depth":[40],"(number":[41,46],"layers)":[43],"and":[44,98,156,183,225],"width":[45],"attention":[48,82,99,195],"heads)":[49],"tend":[50],"to":[51,67,73,204,239],"infinity.":[52],"While":[53],"ResNet":[54],"can":[56],"be":[57],"understood":[58],"as":[59],"controlling":[60,68],"neural":[62,70],"ODE,":[63],"transformer":[64,174],"corresponds":[66],"PDE,":[71],"due":[72],"coupling":[75],"multiple":[77],"token":[78,93,116,218],"distributions":[79,94],"through":[80,96,111],"mechanism.":[83],"Our":[84],"model":[86],"features":[87],"two":[88],"types":[89],"measure":[91],"representations:":[92],"evolving":[95],"layers":[97],"parameters":[100],"at":[101],"each":[102],"layer.":[103],"We":[104,152],"establish":[105],"well-posedness":[106],"forward":[109],"pass":[110],"infinitely":[112],"deep":[113],"transformers,":[114],"characterizing":[115],"evolution":[117],"via":[118],"flow":[119,160,237],"maps":[120],"that":[121,199,235],"satisfy":[122],"ODEs":[123],"function":[125],"spaces.":[126],"Using":[127],"adjoint":[128,146],"sensitivity":[129],"analysis,":[130],"we":[131,197,233],"derive":[132],"an":[133],"explicit":[134],"formula":[135],"conditional":[138,164],"Wasserstein":[139,165],"gradient":[140,159,236],"risk,":[144],"involving":[145],"variables":[147],"governed":[148],"by":[149,216],"backward":[150],"ODEs.":[151],"prove":[153,234],"existence":[155],"uniqueness":[157],"curves":[161],"metric":[166],"space,":[167],"establishing":[168],"foundation":[171],"training.":[175],"A":[176],"key":[177],"technical":[178],"contribution":[179],"is":[180,202,246],"providing":[181],"necessary":[182],"sufficient":[184],"conditions":[185],"injectivity":[187,201,231],"Neural":[190],"Tangent":[191],"Kernel":[192],"(NTK)":[193],"mechanisms:":[196],"show":[198],"NTK":[200,230],"equivalent":[203],"linear":[205],"independence":[206],"log-sum-exp":[208],"functions":[209],"modulo":[210],"affine":[211],"functions,":[212],"condition":[214],"satisfied":[215],"diverse":[217],"distributions,":[219,222,224],"including":[220],"discrete":[221],"uniform":[223],"Gaussian":[226],"mixtures.":[227],"Under":[228],"this":[229],"assumption,":[232],"converges":[238],"global":[240],"minima":[241,252],"when":[242],"initial":[244],"loss":[245],"sufficiently":[247],"small,":[248],"eliminating":[249],"spurious":[250],"local":[251],"from":[253],"optimization":[255],"landscape.":[256]},"counts_by_year":[],"updated_date":"2026-07-01T06:00:48.157686","created_date":"2026-05-20T00:00:00"}