{"id":"https://openalex.org/W7160839712","doi":"https://doi.org/10.48550/arxiv.2605.07959","title":"Convergent Stochastic Training of Attention and Understanding LoRA","display_name":"Convergent Stochastic Training of Attention and Understanding LoRA","publication_year":2026,"publication_date":"2026-05-08","ids":{"openalex":"https://openalex.org/W7160839712","doi":"https://doi.org/10.48550/arxiv.2605.07959"},"language":null,"primary_location":{"id":"pmh:oai:pure.atira.dk:publications/75cc68b1-4b76-40a3-a223-66f4a0267567","is_oa":true,"landing_page_url":"https://research.manchester.ac.uk/en/publications/75cc68b1-4b76-40a3-a223-66f4a0267567","pdf_url":"https://pure.manchester.ac.uk/ws/files/1948660640/2605.07959v1.pdf","source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sun, Z, Kumar, D, Frangi, A F, Mukherjee, A & Sun, M 2026 'Convergent Stochastic Training of Attention and Understanding LoRA'.","raw_type":"info:eu-repo/semantics/preprint"},"type":"preprint","indexed_in":["datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://pure.manchester.ac.uk/ws/files/1948660640/2605.07959v1.pdf","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5061484974","display_name":"Z X Sun","orcid":"https://orcid.org/0009-0003-4127-9925"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Zhengkai","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5021861102","display_name":"Dibyakanti Kumar","orcid":"https://orcid.org/0009-0009-7864-5865"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Kumar, Dibyakanti","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5135865231","display_name":"Alejandro F Frangi","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Frangi, Alejandro F","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"middle","author":{"id":"https://openalex.org/A5084835559","display_name":"Anirbit Mukherjee","orcid":"https://orcid.org/0000-0001-5189-8939"},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Mukherjee, Anirbit","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]},{"author_position":"last","author":{"id":"https://openalex.org/A5135863794","display_name":"Mingfei Sun","orcid":null},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Sun, Mingfei","raw_affiliation_strings":[],"raw_orcid":null,"affiliations":[]}],"institutions":[],"countries_distinct_count":0,"institutions_distinct_count":0,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.6256999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.6256999969482422,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10036","display_name":"Advanced Neural Network Applications","score":0.10360000282526016,"subfield":{"id":"https://openalex.org/subfields/1707","display_name":"Computer Vision and Pattern Recognition"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T11307","display_name":"Domain Adaptation and Few-Shot Learning","score":0.07119999825954437,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/artificial-neural-network","display_name":"Artificial neural network","score":0.4650000035762787},{"id":"https://openalex.org/keywords/training-set","display_name":"Training set","score":0.42590001225471497},{"id":"https://openalex.org/keywords/training","display_name":"Training (meteorology)","score":0.3889000117778778},{"id":"https://openalex.org/keywords/regression","display_name":"Regression","score":0.35989999771118164},{"id":"https://openalex.org/keywords/adaptation","display_name":"Adaptation (eye)","score":0.34150001406669617},{"id":"https://openalex.org/keywords/transformer","display_name":"Transformer","score":0.3034999966621399},{"id":"https://openalex.org/keywords/rank","display_name":"Rank (graph theory)","score":0.30000001192092896}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.5860999822616577},{"id":"https://openalex.org/C154945302","wikidata":"https://www.wikidata.org/wiki/Q11660","display_name":"Artificial intelligence","level":1,"score":0.5059999823570251},{"id":"https://openalex.org/C119857082","wikidata":"https://www.wikidata.org/wiki/Q2539","display_name":"Machine learning","level":1,"score":0.46790000796318054},{"id":"https://openalex.org/C50644808","wikidata":"https://www.wikidata.org/wiki/Q192776","display_name":"Artificial neural network","level":2,"score":0.4650000035762787},{"id":"https://openalex.org/C51632099","wikidata":"https://www.wikidata.org/wiki/Q3985153","display_name":"Training set","level":2,"score":0.42590001225471497},{"id":"https://openalex.org/C2777211547","wikidata":"https://www.wikidata.org/wiki/Q17141490","display_name":"Training (meteorology)","level":2,"score":0.3889000117778778},{"id":"https://openalex.org/C83546350","wikidata":"https://www.wikidata.org/wiki/Q1139051","display_name":"Regression","level":2,"score":0.35989999771118164},{"id":"https://openalex.org/C139807058","wikidata":"https://www.wikidata.org/wiki/Q352374","display_name":"Adaptation (eye)","level":2,"score":0.34150001406669617},{"id":"https://openalex.org/C66322947","wikidata":"https://www.wikidata.org/wiki/Q11658","display_name":"Transformer","level":3,"score":0.3034999966621399},{"id":"https://openalex.org/C164226766","wikidata":"https://www.wikidata.org/wiki/Q7293202","display_name":"Rank (graph theory)","level":2,"score":0.30000001192092896},{"id":"https://openalex.org/C2780428219","wikidata":"https://www.wikidata.org/wiki/Q16952335","display_name":"Cover (algebra)","level":2,"score":0.2865999937057495},{"id":"https://openalex.org/C2984842247","wikidata":"https://www.wikidata.org/wiki/Q197536","display_name":"Deep neural networks","level":3,"score":0.2775000035762787},{"id":"https://openalex.org/C120936955","wikidata":"https://www.wikidata.org/wiki/Q2155640","display_name":"Empirical research","level":2,"score":0.27250000834465027},{"id":"https://openalex.org/C33923547","wikidata":"https://www.wikidata.org/wiki/Q395","display_name":"Mathematics","level":0,"score":0.27129998803138733},{"id":"https://openalex.org/C8272713","wikidata":"https://www.wikidata.org/wiki/Q176737","display_name":"Stochastic process","level":2,"score":0.2646999955177307},{"id":"https://openalex.org/C152877465","wikidata":"https://www.wikidata.org/wiki/Q208042","display_name":"Regression analysis","level":2,"score":0.2572000026702881},{"id":"https://openalex.org/C45555294","wikidata":"https://www.wikidata.org/wiki/Q28113351","display_name":"Inequality","level":2,"score":0.2522999942302704}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:pure.atira.dk:publications/75cc68b1-4b76-40a3-a223-66f4a0267567","is_oa":true,"landing_page_url":"https://research.manchester.ac.uk/en/publications/75cc68b1-4b76-40a3-a223-66f4a0267567","pdf_url":"https://pure.manchester.ac.uk/ws/files/1948660640/2605.07959v1.pdf","source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sun, Z, Kumar, D, Frangi, A F, Mukherjee, A & Sun, M 2026 'Convergent Stochastic Training of Attention and Understanding LoRA'.","raw_type":"info:eu-repo/semantics/preprint"},{"id":"doi:10.48550/arxiv.2605.07959","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2605.07959","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":"cc-by","license_id":"https://openalex.org/licenses/cc-by","version":null,"is_accepted":false,"is_published":null,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:pure.atira.dk:publications/75cc68b1-4b76-40a3-a223-66f4a0267567","is_oa":true,"landing_page_url":"https://research.manchester.ac.uk/en/publications/75cc68b1-4b76-40a3-a223-66f4a0267567","pdf_url":"https://pure.manchester.ac.uk/ws/files/1948660640/2605.07959v1.pdf","source":{"id":"https://openalex.org/S4306400662","display_name":"Research Explorer (The University of Manchester)","issn_l":null,"issn":null,"is_oa":false,"is_in_doaj":false,"is_core":false,"host_organization":"https://openalex.org/I28407311","host_organization_name":"University of Manchester","host_organization_lineage":["https://openalex.org/I28407311"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":"Sun, Z, Kumar, D, Frangi, A F, Mukherjee, A & Sun, M 2026 'Convergent Stochastic Training of Attention and Understanding LoRA'.","raw_type":"info:eu-repo/semantics/preprint"},"sustainable_development_goals":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7160839712.pdf","grobid_xml":"https://content.openalex.org/works/W7160839712.grobid-xml"},"referenced_works_count":0,"referenced_works":[],"related_works":[],"abstract_inverted_index":{"Transformers":[0],"have":[1],"revolutionized":[2],"machine":[3],"learning":[4],"and":[5,80,126],"deploying":[6],"attention":[7,78,125],"layers":[8],"in":[9],"the":[10,72,92,109,112,117,135,138,141],"model":[11],"is":[12,25,39],"increasingly":[13],"standard":[14],"across":[15],"a":[16,34,43,52,77,83,104],"myriad":[17],"of":[18,37,59,122,140],"applications.":[19],"Further,":[20],"for":[21,68,91],"large":[22],"models,":[23],"it":[24,97],"common":[26],"to":[27,41],"implement":[28],"Low":[29],"Rank":[30],"Adaptation":[31],"(LoRA),":[32],"whereby":[33],"factorized":[35],"parameterization":[36],"them":[38],"trained,":[40],"achieve":[42],"surprisingly":[44],"beneficial":[45],"accuracy-size":[46],"trade-off.":[47],"In":[48,115],"this":[49],"work,":[50],"via":[51,99],"unified":[53],"framework":[54],"we":[55],"rigorously":[56],"establish":[57],"trainability":[58,123],"such":[60],"models":[61],"under":[62],"stochastic":[63],"methods.":[64],"We":[65],"prove":[66],"that":[67,103],"any":[69,132],"mild":[70],"regularization,":[71],"empirical":[73],"regression":[74],"loss":[75],"on":[76,82,124,131,134],"layer":[79],"LoRA":[81],"shallow":[84],"neural":[85],"net,":[86],"both":[87,116],"induce":[88],"Poincar\u00e9":[89],"inequality":[90],"corresponding":[93,113],"Gibbs'":[94],"measure.":[95],"Then":[96],"follows":[98],"invoking":[100],"recent":[101],"results":[102,121],"certain":[105],"SDE,":[106],"which":[107],"mimics":[108],"SGD,":[110],"minimizes":[111],"losses.":[114],"cases,":[118],"our":[119],"first-of-its-kind":[120],"nets,":[127],"do":[128],"not":[129],"rely":[130],"assumptions":[133],"data":[136],"or":[137],"size":[139],"architecture.":[142]},"counts_by_year":[],"updated_date":"2026-08-13T07:04:57.449891","created_date":"2026-05-12T00:00:00"}