{"id":"https://openalex.org/W7128539733","doi":"https://doi.org/10.48550/arxiv.2602.08007","title":"From $O(mn)$ to $O(r^2)$: Two-Sided Low-Rank Communication for Adam in Distributed Training with Memory Efficiency","display_name":"From $O(mn)$ to $O(r^2)$: Two-Sided Low-Rank Communication for Adam in Distributed Training with Memory Efficiency","alternate_titles":[],"publication_year":2026,"publication_date":"2026-02-08","ids":{"openalex":"https://openalex.org/W7128539733","doi":"https://doi.org/10.48550/arxiv.2602.08007"},"language":"en","primary_location":{"id":"pmh:oai:arXiv.org:2602.08007","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.08007","pdf_url":"https://arxiv.org/pdf/2602.08007","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"parent_works":[],"lineage":["https://openalex.org/W7128539733"],"child_works_count":0,"type":"preprint","is_traditional":true,"indexed_in":["arxiv","datacite"],"open_access":{"is_oa":true,"oa_status":"green","oa_url":"https://arxiv.org/pdf/2602.08007","any_repository_has_fulltext":true},"authorships":[{"author_position":"first","author":{"id":"https://openalex.org/A5094008674","display_name":"Sizhe Dang","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Dang, Sizhe","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5121690020","display_name":"Jiaqi Shao","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Shao, Jiaqi","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125561966","display_name":"Xiaodong Zheng","orcid":null,"observed_orcids":[]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Zheng, Xiaodong","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5102152323","display_name":"Guang Dai","orcid":"https://orcid.org/0000-0002-3529-9087","observed_orcids":["https://orcid.org/0000-0002-3529-9087"]},"institutions":[],"countries":[],"is_corresponding":false,"raw_author_name":"Dai, Guang","raw_affiliation_strings":["SGIT AI Lab"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"SGIT AI Lab","institution_ids":[]}]},{"author_position":"middle","author":{"id":"https://openalex.org/A5125531872","display_name":"Yan Song","orcid":"https://orcid.org/0009-0002-5919-739X","observed_orcids":["https://orcid.org/0009-0002-5919-739X"]},"institutions":[{"id":"https://openalex.org/I126520041","display_name":"University of Science and Technology of China","ror":"https://ror.org/04c4dkn09","country_code":"CN","type":"education","lineage":["https://openalex.org/I126520041","https://openalex.org/I19820366"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Song, Yan","raw_affiliation_strings":["University of Science and Technology of China"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"University of Science and Technology of China","institution_ids":["https://openalex.org/I126520041"]}]},{"author_position":"last","author":{"id":"https://openalex.org/A5029403877","display_name":"Haishan Ye","orcid":"https://orcid.org/0000-0002-0242-4857","observed_orcids":["https://orcid.org/0000-0002-0242-4857","https://orcid.org/0000-0001-9361-4906"]},"institutions":[{"id":"https://openalex.org/I87445476","display_name":"Xi'an Jiaotong University","ror":"https://ror.org/017zhmm22","country_code":"CN","type":"education","lineage":["https://openalex.org/I87445476"]}],"countries":["CN"],"is_corresponding":false,"raw_author_name":"Ye, Haishan","raw_affiliation_strings":["Xi'an Jiaotong University"],"raw_orcid":null,"affiliations":[{"raw_affiliation_string":"Xi'an Jiaotong University","institution_ids":["https://openalex.org/I87445476"]}]}],"institutions":[],"countries_distinct_count":1,"institutions_distinct_count":2,"corresponding_author_ids":[],"corresponding_institution_ids":[],"apc_list":null,"apc_paid":null,"fwci":null,"has_fulltext":true,"cited_by_count":0,"citation_normalized_percentile":null,"cited_by_percentile_year":null,"biblio":{"volume":null,"issue":null,"first_page":null,"last_page":null},"is_retracted":false,"is_paratext":false,"is_xpac":false,"primary_topic":{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},"topics":[{"id":"https://openalex.org/T11612","display_name":"Stochastic Gradient Optimization Techniques","score":0.9945999979972839,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10054","display_name":"Parallel Computing and Optimization Techniques","score":0.002099999925121665,"subfield":{"id":"https://openalex.org/subfields/1708","display_name":"Hardware and Architecture"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}},{"id":"https://openalex.org/T10028","display_name":"Topic Modeling","score":0.00019999999494757503,"subfield":{"id":"https://openalex.org/subfields/1702","display_name":"Artificial Intelligence"},"field":{"id":"https://openalex.org/fields/17","display_name":"Computer Science"},"domain":{"id":"https://openalex.org/domains/3","display_name":"Physical Sciences"}}],"keywords":[{"id":"https://openalex.org/keywords/adam-optimizer","display_name":"Adam optimizer","score":0.9120000004768372},{"id":"https://openalex.org/keywords/distributed-training","display_name":"distributed training","score":0.7559999823570251},{"id":"https://openalex.org/keywords/gradient-synchronization","display_name":"gradient synchronization","score":0.753000020980835},{"id":"https://openalex.org/keywords/memory-efficiency","display_name":"memory efficiency","score":0.6710000038146973},{"id":"https://openalex.org/keywords/bidirectional-communication","display_name":"two-way communication","score":0.703000009059906},{"id":"https://openalex.org/keywords/randomized-svd","display_name":"randomized SVD","score":0.7350000143051147},{"id":"https://openalex.org/keywords/communication-efficiency","display_name":"communication efficiency","score":0.36500000953674316}],"concepts":[{"id":"https://openalex.org/C41008148","wikidata":"https://www.wikidata.org/wiki/Q21198","display_name":"Computer science","level":0,"score":0.7577000260353088},{"id":"https://openalex.org/C2778562939","wikidata":"https://www.wikidata.org/wiki/Q1298791","display_name":"Synchronization (alternating current)","level":3,"score":0.659600019454956},{"id":"https://openalex.org/C134066672","wikidata":"https://www.wikidata.org/wiki/Q1424639","display_name":"Payload (computing)","level":3,"score":0.6134999990463257},{"id":"https://openalex.org/C43364308","wikidata":"https://www.wikidata.org/wiki/Q8799","display_name":"Byte","level":2,"score":0.5945000052452087},{"id":"https://openalex.org/C26517878","wikidata":"https://www.wikidata.org/wiki/Q228039","display_name":"Key (lock)","level":2,"score":0.5519000291824341},{"id":"https://openalex.org/C162932704","wikidata":"https://www.wikidata.org/wiki/Q1058791","display_name":"Synchronizing","level":3,"score":0.5203999876976013},{"id":"https://openalex.org/C2781238097","wikidata":"https://www.wikidata.org/wiki/Q175026","display_name":"Object (grammar)","level":2,"score":0.4291999936103821},{"id":"https://openalex.org/C41608201","wikidata":"https://www.wikidata.org/wiki/Q980509","display_name":"Embedding","level":2,"score":0.41620001196861267},{"id":"https://openalex.org/C2776760102","wikidata":"https://www.wikidata.org/wiki/Q5139990","display_name":"Code (set theory)","level":3,"score":0.40389999747276306},{"id":"https://openalex.org/C125411270","wikidata":"https://www.wikidata.org/wiki/Q18653","display_name":"Encoding (memory)","level":2,"score":0.3668000102043152},{"id":"https://openalex.org/C2777145635","wikidata":"https://www.wikidata.org/wiki/Q515636","display_name":"FIFO (computing and electronics)","level":2,"score":0.3626999855041504},{"id":"https://openalex.org/C179254644","wikidata":"https://www.wikidata.org/wiki/Q13222844","display_name":"Moment (physics)","level":2,"score":0.36250001192092896},{"id":"https://openalex.org/C158156997","wikidata":"https://www.wikidata.org/wiki/Q1416645","display_name":"Models of communication","level":2,"score":0.3610999882221222},{"id":"https://openalex.org/C120314980","wikidata":"https://www.wikidata.org/wiki/Q180634","display_name":"Distributed computing","level":1,"score":0.3601999878883362},{"id":"https://openalex.org/C79403827","wikidata":"https://www.wikidata.org/wiki/Q3988","display_name":"Real-time computing","level":1,"score":0.35600000619888306},{"id":"https://openalex.org/C91481028","wikidata":"https://www.wikidata.org/wiki/Q1054686","display_name":"Distributed memory","level":3,"score":0.3440000116825104},{"id":"https://openalex.org/C2164484","wikidata":"https://www.wikidata.org/wiki/Q5170150","display_name":"Core (optical fiber)","level":2,"score":0.34380000829696655},{"id":"https://openalex.org/C48677424","wikidata":"https://www.wikidata.org/wiki/Q6888088","display_name":"Mode (computer interface)","level":2,"score":0.3384000062942505},{"id":"https://openalex.org/C12269588","wikidata":"https://www.wikidata.org/wiki/Q132364","display_name":"Communications protocol","level":2,"score":0.32829999923706055},{"id":"https://openalex.org/C111335779","wikidata":"https://www.wikidata.org/wiki/Q3454686","display_name":"Reduction (mathematics)","level":2,"score":0.328000009059906},{"id":"https://openalex.org/C101765175","wikidata":"https://www.wikidata.org/wiki/Q577764","display_name":"Communications system","level":2,"score":0.31940001249313354},{"id":"https://openalex.org/C37829124","wikidata":"https://www.wikidata.org/wiki/Q763575","display_name":"Parallel communication","level":3,"score":0.314300000667572},{"id":"https://openalex.org/C2781215313","wikidata":"https://www.wikidata.org/wiki/Q3493345","display_name":"SPARK (programming language)","level":2,"score":0.3046000003814697},{"id":"https://openalex.org/C2778112365","wikidata":"https://www.wikidata.org/wiki/Q3511065","display_name":"Sequence (biology)","level":2,"score":0.3041999936103821},{"id":"https://openalex.org/C9390403","wikidata":"https://www.wikidata.org/wiki/Q3966","display_name":"Computer hardware","level":1,"score":0.2922999858856201},{"id":"https://openalex.org/C151319957","wikidata":"https://www.wikidata.org/wiki/Q752739","display_name":"Asynchronous communication","level":2,"score":0.28290000557899475},{"id":"https://openalex.org/C113775141","wikidata":"https://www.wikidata.org/wiki/Q428691","display_name":"Computer engineering","level":1,"score":0.26919999718666077},{"id":"https://openalex.org/C32834561","wikidata":"https://www.wikidata.org/wiki/Q660730","display_name":"Subspace topology","level":2,"score":0.26109999418258667},{"id":"https://openalex.org/C761482","wikidata":"https://www.wikidata.org/wiki/Q118093","display_name":"Transmission (telecommunications)","level":2,"score":0.2529999911785126},{"id":"https://openalex.org/C149635348","wikidata":"https://www.wikidata.org/wiki/Q193040","display_name":"Embedded system","level":1,"score":0.2515999972820282}],"mesh":[],"locations_count":2,"locations":[{"id":"pmh:oai:arXiv.org:2602.08007","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.08007","pdf_url":"https://arxiv.org/pdf/2602.08007","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},{"id":"doi:10.48550/arxiv.2602.08007","doi":"https://doi.org/10.48550/arxiv.2602.08007","is_oa":true,"landing_page_url":"https://doi.org/10.48550/arxiv.2602.08007","pdf_url":null,"source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":null,"is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"Preprint"}],"best_oa_location":{"id":"pmh:oai:arXiv.org:2602.08007","doi":null,"is_oa":true,"landing_page_url":"http://arxiv.org/abs/2602.08007","pdf_url":"https://arxiv.org/pdf/2602.08007","source":{"id":"https://openalex.org/S4306400194","display_name":"arXiv (Cornell University)","issn_l":"2331-8422","issn":["2331-8422"],"is_oa":true,"is_in_doaj":false,"is_core":false,"listed_in":[],"host_organization":"https://openalex.org/I205783295","host_organization_name":"Cornell University","host_organization_lineage":["https://openalex.org/I205783295"],"host_organization_lineage_names":[],"type":"repository"},"license":null,"license_id":null,"version":"submittedVersion","is_accepted":false,"is_published":false,"raw_source_name":null,"raw_type":"text"},"sustainable_development_goals":[],"study_designs":[],"awards":[],"funders":[],"has_content":{"grobid_xml":true,"pdf":true},"content_urls":{"pdf":"https://content.openalex.org/works/W7128539733.pdf","grobid_xml":"https://content.openalex.org/works/W7128539733.grobid-xml"},"referenced_works_count":26,"referenced_works":["https://openalex.org/W2774000609","https://openalex.org/W2911863041","https://openalex.org/W2923014074","https://openalex.org/W2946900926","https://openalex.org/W2965373594","https://openalex.org/W2973727699","https://openalex.org/W2981852735","https://openalex.org/W3081168214","https://openalex.org/W3127394918","https://openalex.org/W3129831491","https://openalex.org/W3187255235","https://openalex.org/W3195577433","https://openalex.org/W3202388704","https://openalex.org/W3205803342","https://openalex.org/W4321636690","https://openalex.org/W4330338093","https://openalex.org/W4384918448","https://openalex.org/W4386768656","https://openalex.org/W4392575839","https://openalex.org/W4403573844","https://openalex.org/W4404089139","https://openalex.org/W4415797226","https://openalex.org/W4417170051","https://openalex.org/W7131243825","https://openalex.org/W7133233472","https://openalex.org/W7197021544"],"related_works":["https://openalex.org/W7127955665","https://openalex.org/W2946900926","https://openalex.org/W4405031739","https://openalex.org/W4221154412","https://openalex.org/W4404308019","https://openalex.org/W4404089343","https://openalex.org/W4416048468","https://openalex.org/W4417463957","https://openalex.org/W7133235158","https://openalex.org/W4386804004"],"abstract_inverted_index":{"As":[0],"foundation":[1],"models":[2],"continue":[3],"to":[4,65,85,118,141],"scale,":[5],"pretraining":[6,138],"increasingly":[7],"relies":[8],"on":[9,155],"data-parallel":[10],"distributed":[11],"optimization,":[12],"making":[13],"bandwidth-limited":[14],"gradient":[15,48],"synchronization":[16,37],"a":[17,71,105,170],"key":[18],"bottleneck.":[19],"Orthogonally,":[20],"projection-based":[21],"low-rank":[22,63,116],"optimizers":[23],"were":[24],"mainly":[25],"designed":[26],"for":[27,33,43,174],"memory":[28,131],"efficiency,":[29],"but":[30],"remain":[31],"suboptimal":[32],"communication-limited":[34],"training:":[35],"one-sided":[36],"still":[38],"transmits":[39],"an":[40,44],"$O(rn)$":[41],"object":[42],"$m\\times":[45],"n$":[46],"matrix":[47],"and":[49,124,130,154],"refresh":[50,108,125],"steps":[51],"can":[52],"dominate":[53],"peak":[54,98],"communicated":[55,148],"bytes.":[56],"We":[57,113],"propose":[58],"TSR,":[59],"which":[60],"brings":[61],"two-sided":[62],"communication":[64,99,117,129,160],"Adam-family":[66],"updates":[67],"(TSR-Adam)":[68],"by":[69,152,161],"synchronizing":[70],"compact":[72],"core":[73],"$U^\\top":[74],"G":[75],"V\\in\\mathbb{R}^{r\\times":[76],"r}$,":[77],"reducing":[78],"the":[79,97,175],"dominant":[80],"per-step":[81],"payload":[82],"from":[83,100,139],"$O(mn)$":[84],"$O(r^2)$":[86],"while":[87,163],"keeping":[88,134],"moment":[89],"states":[90],"in":[91],"low-dimensional":[92],"cores.":[93],"To":[94],"further":[95,168],"reduce":[96],"subspace":[101],"refresh,":[102],"TSR-Adam":[103,145],"adopts":[104],"randomized":[106],"SVD-based":[107],"that":[109],"avoids":[110],"full-gradient":[111],"synchronization.":[112],"additionally":[114],"extend":[115],"embedding":[119],"gradients":[120],"with":[121],"embedding-specific":[122],"ranks":[123],"schedules,":[126],"yielding":[127],"additional":[128],"savings":[132],"over":[133],"embeddings":[135],"dense.":[136],"Across":[137],"60M":[140],"1B":[142],"model":[143],"scales,":[144],"reduces":[146,159],"average":[147],"bytes":[149],"per":[150],"step":[151],"$13\\times$,":[153],"GLUE":[156],"fine-tuning":[157],"it":[158],"$25\\times$,":[162],"achieving":[164],"comparable":[165],"performance;":[166],"we":[167],"provide":[169],"theoretical":[171],"stationarity":[172],"analysis":[173],"proposed":[176],"update.":[177],"Code":[178],"is":[179],"available":[180],"at":[181],"https://github.com/DKmiyan/TSR-Adam.":[182]},"cited_by_api_url":"https://api.openalex.org/works?filter=cites:W7128539733","counts_by_year":[],"updated_date":"2026-11-16T12:05:36.265796","created_date":"2026-02-11T00:00:00"}
